Official Code Repo for TIGERScore
from setuptools import setup, find_packages

description = """
TIGERScore, a Trained metric that follows Instruction Guidance to perform Explainable, and Reference-free evaluation over a wide spectrum of text generation tasks. 
Different from other automatic evaluation methods that only provide arcane scores, TIGERScore is guided by the natural language instruction to provide error analysis to pinpoint the mistakes in the generated text.
"""

# transformers==4.33.2
# datasets==2.10.0
# torch
# accelerate
# wget
# pycocoevalcap
# spacy
# evaluate
# prettytable
# fairscale
# bert_score
# gdcm 
# pydicom
# sacremoses
# apache_beam
# deepspeed
# bitsandbytes
# openai
# nltk
# scipy
# json5
# peft
# fire
# gradio
# sentencepiece
# tiktoken
# dacite
# wandb
# rouge_score
# bs4
# py7zr
# sacrebleu
# gdown
# git+https://github.com/google-research/mt-metrics-eval.git

setup(
    name='tigerscore',
    version='0.0.1',
    description=description,
    author='Dongfu Jiang',
    author_email='dongfu.jiang@uwaterloo.ca',
    packages=find_packages(),
    url='https://tiger-ai-lab.github.io/TIGERScore/',
    install_requires=[
        'transformers>=4.33.2',
        'datasets>=2.10.0',
        'torch',
        'accelerate',
        'wget',
        'pycocoevalcap',
        'spacy',
        'evaluate',
        'prettytable',
        'fairscale',
        'bert_score',
        'gdcm',
        'pydicom',
        'sacremoses',
        'apache_beam',
        'deepspeed',
        'bitsandbytes',
        'openai',
        'nltk',
        'scipy',
        'json5',
        'peft',
        'fire',
        'gradio',
        'sentencepiece',
        'tiktoken',
        'dacite',
        'wandb',
        'rouge_score',
        'bs4',
        'py7zr',
        'sacrebleu',
        'gdown',
        'accelerate',
        'bitsandbytes',
        'mt-metrics-eval @ git+https://github.com/google-research/mt-metrics-eval.git',
    ],
)

setup.py
TIGER-AI-Lab/TIGERScore
from tigerscore.scorer.tigerscore import TIGERScorer
tigerscore/__init__.py
TIGER-AI-Lab/TIGERScore
# The task in slurm connot support long time download,so just download in shell.
from model_utils import (
    build_model,
    build_tokenizer
)
import argparse
import os
from pathlib import Path


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--models', type = str, default = None)
    parser.add_argument('--model_type', type = str, default = None)
    parser.add_argument('--cache_dir', type = str, default = None)
    args = parser.parse_args()
    if args.cache_dir is None:
        args.cache_dir = Path(os.path.abspath(__file__)).parent.parent.parent / "hf_models"
    
    for model in args.models.split(','):
        tokenizer = build_tokenizer(model,cache_dir=args.cache_dir,resume_download=True,trust_remote_code=True)
        model = build_model(args.model_type,model,cache_dir=args.cache_dir,resume_download=True,trust_remote_code=True)
        
tigerscore/candidates_generation/downmodel.py
TIGER-AI-Lab/TIGERScore
"""
    This file is taken from This file is modified based on:
    https://github.com/Ravoxsg/SummaReranker-ACL-22-/blob/main/src/candidate_generation/engine.py
    We thank the authors for sharing their code.
"""
import gc
from tqdm import tqdm
import torch
import numpy as np
import torch.nn.functional as F
from typing import List, Dict, Tuple
def beam_search_step(inputs: Dict, tokenizer, base_model, args, **kwargs):
    kwargs['return_dict_in_generate'] = True
    kwargs['output_scores'] = True
    # 1 - beam search
    if args.decoding_method == "beam_search":
        outputs = base_model.generate(
            **inputs,
            num_beams = args.num_beams,
            num_return_sequences = args.num_return_sequences,
            max_new_tokens = args.output_max_length,
            repetition_penalty = args.repetition_penalty,
            length_penalty = args.length_penalty,
            no_repeat_ngram_size = args.no_repeat_ngram_size,
            use_cache = True,
            early_stopping = True,
            temperature = args.temperature,
            **kwargs
        )
    # 2 - diverse beam search
    if args.decoding_method == "diverse_beam_search":
        outputs = base_model.generate(
            **inputs,
            num_beams = args.num_beams,
            num_beam_groups = args.num_beam_groups,
            num_return_sequences = args.num_return_sequences,
            max_new_tokens = args.output_max_length,
            diversity_penalty = args.diversity_penalty,
            repetition_penalty = args.repetition_penalty,
            length_penalty = args.length_penalty,
            no_repeat_ngram_size = args.no_repeat_ngram_size,
            use_cache = True,
            early_stopping = True,
            temperature = args.temperature,
            **kwargs
        )
    # 3 - top-p sampling
    if args.decoding_method == "top_p_sampling":
        outputs = base_model.generate(
            **inputs,
            num_beams = 1,
            do_sample = True,
            top_p = args.top_p,
            num_return_sequences = args.num_return_sequences,
            max_new_tokens = args.output_max_length,
            repetition_penalty = args.repetition_penalty,
            length_penalty = args.length_penalty,
            no_repeat_ngram_size = args.no_repeat_ngram_size,
            use_cache = True,
            early_stopping = True,
            temperature = args.temperature,
            **kwargs
        )
    # 4 - top-k sampling
    if args.decoding_method == "top_k_sampling":
        outputs = base_model.generate(
            **inputs,
            num_beams = 1,
            do_sample = True,
            top_k = args.top_k,
            num_return_sequences = args.num_return_sequences,
            max_new_tokens = args.output_max_length,
            repetition_penalty = args.repetition_penalty,
            length_penalty = args.length_penalty,
            no_repeat_ngram_size = args.no_repeat_ngram_size,
            use_cache = True,
            early_stopping = True,
            temperature = args.temperature,
            **kwargs
        )
    masked_logits = torch.stack(outputs.scores, dim=0) # for top-p and top-k sampling, some scores will be masked as -inf. These scores are not processed by softmax and logrithm.
    masked_logits = F.log_softmax(masked_logits, dim=1)
    summary_ids = outputs.sequences
    logprobs = []
    # Different process for decoder-only models and encoder-decoder models
    if "input_ids" in inputs and \
        summary_ids.shape[1] == inputs['input_ids'].shape[1] + masked_logits.shape[0]:
        # for decoder-only models
        summary_ids = summary_ids[:, inputs['input_ids'].shape[1]:] # remove input_ids
        for i in range(summary_ids.shape[0]):
            logprobs.append([])
            for j in range(summary_ids.shape[1]): # token_idx
                if summary_ids[i][j] == tokenizer.eos_token_id:
                    break
                logprobs[i].append(masked_logits[j, i, summary_ids[i][j]].item())
    else:
        # for encoder-decoder models
        for i in range(summary_ids.shape[0]):
            logprobs.append([])
            # shift of decoder because of the additional bos_token
            for j in range(summary_ids.shape[1] - 1): # token_idx
                if summary_ids[i][j+1] == tokenizer.eos_token_id:
                    break
                logprobs[i].append(masked_logits[j, i, summary_ids[i][j+1]].item())

    logprobs = [sum(_probs) for _probs in logprobs]
    generated = tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
    del summary_ids
    gc.collect()

    batch_generated = []
    batch_logprobs = []
    bz = list(inputs.values())[0].shape[0]
    for i in range(bz):
        batch_generated.append(generated[i*args.num_return_sequences:(i+1)*args.num_return_sequences])
        batch_logprobs.append(logprobs[i*args.num_return_sequences:(i+1)*args.num_return_sequences])
    return {
        "generated": batch_generated,
        "logprobs": batch_logprobs
    }

tigerscore/candidates_generation/engine.py
TIGER-AI-Lab/TIGERScore
"""
    Eval results will be continuously saved to ../../data/prepared/{dataset_name}/{set_name}/dataset.jsonl
"""
import argparse
import sys
import os
import psutil
import json
import random
import numpy as np
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from common.utils import (
    seed_everything,
    str2bool,
)
from common.evaluation import (
    overall_eval,
    SUPPORTED_METRICS
)
from pathlib import Path

def save_prepared(
    dataset,
    set_name,
    data_dir,
):
    ds_path = Path(data_dir) / dataset / f"{set_name}_data.json"
    save_prepared_path = Path(data_dir) / dataset / f"{set_name}_data_prepared.json"
    assert ds_path.exists(), f"{ds_path} does not exist"
    with open(ds_path) as f:
        ds_data = json.load(f)
    # load candidates
    candidates_dir = Path(data_dir) / dataset / "candidates" / set_name 
    decoding_method_dirs = [x for x in candidates_dir.iterdir() if x.is_dir()]
    for decoding_method_dir in decoding_method_dirs:
        decoding_method = decoding_method_dir.name
        # load candidates with eval scores
        candidate_eval_files = [x for x in decoding_method_dir.iterdir() if x.is_file() and x.suffixes[-2:] == [".eval", ".json"]]
        for candidate_eval_file in candidate_eval_files:
            model_name = candidate_eval_file.stem.split(".")[0]
            with open(candidate_eval_file) as f:
                eval_candidates = json.load(f)
                eval_candidates = {x["id"]: x["candidates"] for x in eval_candidates}
            assert set(eval_candidates.keys()) == set([x["id"] for x in ds_data]), \
                f"candidate ids do not match for {dataset} {set_name} {decoding_method} {model_name}"
            for example in ds_data:
                example_id = example["id"]
                if "candidates" not in example:
                    example["candidates"] = []

                target_candidates = eval_candidates[example_id]
                target_candidates = sorted(target_candidates, key=lambda x: x["scores"]["bleu"], reverse=True)
                # target_candidate = random.choice(target_candidates)
                # target_candidate = target_candidates[0]
                # example["candidates"].append({
                #     "decoding_method": decoding_method,
                #     "model": model_name,
                #     "text": target_candidate["text"],
                #     "scores": target_candidate["scores"],
                #     }
                # )

                for eval_candidate in eval_candidates[example_id]:
                    example["candidates"].append({
                        "decoding_method": decoding_method,
                        "model": model_name,
                        "text": eval_candidate["text"],
                        "scores": eval_candidate["scores"],
                    })
    print(f"Total no. of {set_name} examples in the aggregated dataset: {len(ds_data)}")
    with open(save_prepared_path, "w") as f:
        json.dump(ds_data, f, indent=4, ensure_ascii=False)
    print(f"Saved aggregated {set_name} data to {save_prepared_path}")

def main(args):
    # seed
    seed_everything(args.seed)

    # prepare metrics
    if 'rouge' in args.metrics:
        args.metrics.extend(["rouge1", "rouge2", "rougeL", "rougeLsum"])
        args.metrics.remove('rouge')
    metrics = args.metrics
    assert set(metrics).issubset(set(SUPPORTED_METRICS)), \
        "Unsupported metrics: {}".format(set(SUPPORTED_METRICS)-set(metrics))

    for dataset in args.datasets:
        dataset = dataset.replace(":", "/")
        
        for set_name in args.sets:
            print("Evaluating dataset: {} \t set: {}".format(dataset, set_name))
            # get all the decoding method
            candidates_dir = Path(args.data_dir) / dataset / "candidates" / set_name
            decoding_methods = [f.name for f in candidates_dir.iterdir() if f.is_dir()]
            if len(decoding_methods) == 0:
                print("No candidates generated for {}-{}".format(dataset, set_name))
                continue
            for decoding_method in decoding_methods:
                print("Decoding method: {}".format(decoding_method))
                candidate_files = [f for f in (candidates_dir / decoding_method).iterdir() if f.is_file() and f.suffix == ".json" and ".eval" not in f.suffixes]
                if len(candidate_files) == 0:
                    print("No candidates generated for {}-{}-{}".format(dataset, set_name, decoding_method))
                    continue
                for candidate_file in candidate_files:
                    print("Model name: {}".format(candidate_file.stem))
                    # load candidates
                    candidate_eval_file = candidate_file.with_suffix(".eval.json")
                    # candidate_eval_file = candidate_file.with_suffix("._eval.json")
                    if not candidate_eval_file.exists() or args.overwrite:
                        print("Create a new eval file: {}".format(candidate_eval_file))
                        candidates = json.load(open(candidate_file, 'r'))
                        # create a new eval file if not exists, seperate from the original candidates file
                        json.dump(candidates, open(candidate_eval_file, 'w'), indent=4, ensure_ascii=False)
                    else:
                        print("Load existing eval file: {}".format(candidate_eval_file))
                        candidates = json.load(open(candidate_eval_file, 'r'))
                    # check if the candidates have already been evaluated (sample check)
                    is_evaluated = True
                    for i, sample in enumerate(candidates):
                        id = sample['id']
                        _sample_cands = sample['candidates']
                        for j, _sample_cand in enumerate(_sample_cands):
                            if not (set(metrics) <= set(_sample_cand['scores'].keys())):
                                is_evaluated = False
                                break
                        if not is_evaluated:
                            break
                    # Evaluate
                    if not is_evaluated or args.overwrite:
                        if is_evaluated:
                            print("Overwrite mode: candidates will be re-evaluated")
                        # load targets and pure candidates
                        DS = json.load(open(Path(args.data_dir) / dataset / f"{set_name}_data.json", 'r'))
                        DS = {x['id']: x for x in DS}
                        pure_candidates = [[x['text'] for x in item['candidates']] for item in candidates]
                        targets = [DS[x['id']]['output'] for x in candidates]
                        # evaluate
                        scores = overall_eval(pure_candidates, targets, metrics, args.num_workers)
                        # save
                        assert set(scores.keys()) == set(metrics)
                        for metric in metrics:
                            scores_metric = scores[metric]
                            for i, sample in enumerate(candidates):
                                id = sample['id']
                                _sample_cands = sample['candidates']
                                for j, _sample_cand in enumerate(_sample_cands):
                                    _sample_cand['scores'][metric] = scores_metric[i][j]
                        json.dump(candidates, open(candidate_eval_file, 'w'), indent=4, ensure_ascii=False)
                        print("Evaluation results saved to {}".format(candidate_eval_file))
                    else:
                        print("Candidates have already been evaluated, skip")
                    # Report the evaluation results
                    for metric in metrics:
                        scores = [[x['scores'][metric] for x in item['candidates']] for item in candidates]
                        scores = np.array(scores)
                        print("Metric: {}".format(metric))
                        print("Average Min Score: {:.3f}".format(scores.min(axis=1).mean()))
                        print("Average Max Score: {:.3f}".format(scores.max(axis=1).mean()))
                        print("Average Mean Score: {:.3f}".format(scores.mean(axis=1).mean()))
                        print("Average Default Top-1 Score: {:.3f}".format(scores[:,0].mean()))
                        print("Average Default Bottom-1 Score: {:.3f}".format(scores[:,-1].mean()))
        print("Done for dataset: {}".format(dataset))

        if args.save_prepared:
            for set_name in args.sets:
                save_prepared(dataset, set_name, args.data_dir)

    print("Done for all datasets: {}".format(args.datasets))

                    


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--data_dir", type=str, default="../../data")
    parser.add_argument("--dataset", type=str, default="cnndm")
    parser.add_argument("--set", type=str, default="test")
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--num_workers", type=int, default=1)
    parser.add_argument("--overwrite", type=str2bool, default=False)
    parser.add_argument("--save_prepared", type=str2bool, default=True,
        help="aggregate the candidates and save them to a single file for each dataset and set")
    # metrics
    parser.add_argument("--metrics", type=str, default="rouge,bleu",
        help="metrics to compute, support rouge, bleu, bleurt, cider, spice, bleu4, bertscore, bartscore")
    args = parser.parse_args()
    args.metrics = args.metrics.split(",")
    args.datasets = args.dataset.split(",")
    args.sets = args.set.split(",")
    print(args)
    main(args)

tigerscore/candidates_generation/eval_candidates.py
TIGER-AI-Lab/TIGERScore
'''
From https://github.com/huggingface/notebooks/blob/main/examples/summarization.ipynb
'''

#!/usr/bin/env python
# coding=utf-8
# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Fine-tuning the library models for sequence to sequence.
"""
# You can also adapt this script on your own sequence to sequence task. Pointers for this are left as comments.
########!!!!!!!!!!!!!########
# set WANDB_DISABLED=True for offline logging
########!!!!!!!!!!!!!########

import logging
import os
import sys
import json

import numpy as np
from datasets import load_dataset
import evaluate
from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
import nltk
import torch
import argparse

import transformers
from transformers import (
    AutoConfig,
    AutoModel,
    AutoTokenizer,
    AutoTokenizer,
    DataCollatorForSeq2Seq,
    HfArgumentParser,
    Seq2SeqTrainingArguments,
    set_seed,
    Trainer,
    TrainingArguments
)
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from common.utils import (
    seed_everything,
    str2bool,
    empty2None,
    empty2Noneint,
    fetch_images,
)
from model_utils import (
    build_model,
    build_tokenizer,
    build_processor,
)
from pathlib import Path
from generate_candidates import get_model_size, get_torch_dtype,GenerationDataset
import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import datasets
import evaluate
import nltk  # Here to have a nice missing dependency error message early on
import numpy as np
from datasets import load_dataset
from filelock import FileLock

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSeq2SeqLM,
    AutoTokenizer,
    DataCollatorForSeq2Seq,
    HfArgumentParser,
    MBart50Tokenizer,
    MBart50TokenizerFast,
    MBartTokenizer,
    MBartTokenizerFast,
    Seq2SeqTrainer,
    Seq2SeqTrainingArguments,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint
from transformers.utils import check_min_version, is_offline_mode, send_example_telemetry
from transformers.utils.versions import require_version

def main(args):
    # seed
    
    seed_everything(args.seed)

    # device
    device = torch.device("cpu")
    if args.cuda and torch.cuda.is_available():
        device = torch.device("cuda")
    args.device = device
    print("\nUsing device {}".format(device))
    
    # tokenizer
    tokenizer = build_tokenizer(args.model, cache_dir=args.cache_dir, resume_download=True)
    
    #dataset
    dataset_name = args.dataset
    train_data_file = Path(args.data_dir) / dataset_name.replace(":", "/") / f"train_data.json"
    val_data_file = Path(args.data_dir) / dataset_name.replace(":", "/") / f"validation_data.json"
    
    
    # train_data = json.load(open(train_data_file, 'r'))
    # val_data = json.load(open(val_data_file, 'r'))
    train_dataset = load_dataset('json', data_files=str(train_data_file), split="train")
    val_dataset = load_dataset('json', data_files=str(val_data_file),split="train")
    
    def preprocess_function(examples):
        max_input_length = args.prompt_max_length
        max_target_length = args.prompt_max_length
        model_inputs = tokenizer(examples['input'], max_length=max_input_length, truncation=True, padding="max_length")
        # Setup the tokenizer for targets
        # label_ids = tokenizer.encode(examples['output'], max_length=max_target_length, truncation=True)
        with tokenizer.as_target_tokenizer():
            labels = tokenizer(examples["output"], max_length=max_target_length, truncation=True, padding="max_length")
        labels["input_ids"] = [
            [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels["input_ids"]
        ]
        model_inputs["labels"] = labels["input_ids"]
        if model_inputs["labels"] == None:
            print("BOOM")
        return model_inputs
    
    tokenized_train_dataset = train_dataset.map(
        preprocess_function,
        batched=True,
    )
    tokenized_val_dataset = val_dataset.map(
        preprocess_function,
        batched=True,
    )
    
       
    
    # print("Dataset loaded: ", dataset)
    metric = evaluate.load(str(args.metric_path)+'/'+args.metric_name+".py")
    # model
    model = build_model(
        args.model_type,
        args.model,
        torch_dtype=get_torch_dtype(args.dtype),
        device_map="auto",
        cache_dir=args.cache_dir, resume_download=True)
    n_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
    print("\nThe {} has {} trainable parameters".format(args.model, get_model_size(n_params)))
        
   

    def compute_metrics(eval_pred):
        logits, labels = eval_pred
        predictions = np.argmax(logits, axis=-1)
        return metric.compute(predictions=predictions, references=labels)
    
    batch_size= 16
    training_args = Seq2SeqTrainingArguments(
        str(os.path.join(args.output_dir, args.output_name)),
        evaluation_strategy = "epoch",
        learning_rate=2e-5,
        per_device_train_batch_size=batch_size,
        per_device_eval_batch_size=batch_size,
        weight_decay=0.01,
        save_total_limit=3,
        num_train_epochs=1,
        predict_with_generate=True,
        fp16=True)
    
    label_pad_token_id = -100
    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, label_pad_token_id=label_pad_token_id,pad_to_multiple_of=8)
    
    trainer =  Seq2SeqTrainer(
        model=model,
        args=training_args,
        train_dataset=tokenized_train_dataset,
        eval_dataset=tokenized_val_dataset,
        data_collator=data_collator,
        compute_metrics=compute_metrics,
    )   
    trainer.train()
    
    
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    
    parser.add_argument('--seed', type = int, default = 42)
    parser.add_argument('--cuda', type = str2bool, default = True)

    # data
    parser.add_argument('--data_dir', type = str, default = '../../data')
    parser.add_argument('--dataset', type = empty2None, required=True)
    # parser.add_argument('--set', type = str, default = "test")
    # parser.add_argument('--max_size', type = int, default = None)
     # model
    parser.add_argument('--model_type', type = str, default = "t5",)
    parser.add_argument('--model', type = str, default = "google/flan-t5-xxl")
    parser.add_argument('--dtype', type = str, default = "float32",
                        choices = ["float32", "float16", "bfloat16", "int8"])
    parser.add_argument('--cache_dir', type = str, default = None)
    parser.add_argument('--prefix', type = empty2None, default = None)
    parser.add_argument('--output_dir', type = str, default = None)
    parser.add_argument('--prompt_max_length', type = int, default = 1024)
    parser.add_argument('--no_instruction', type = str2bool, default = False)
    parser.add_argument('--output_name', type = str, default = None)
    parser.add_argument('--metric_path', type = str, default = None)
    parser.add_argument('--metric_name', type = str, default = 'rouge')
    
    args = parser.parse_args()
    
    if args.cache_dir is None:
        args.cache_dir = Path(os.path.abspath(__file__)).parent.parent.parent / "hf_models"
    if args.output_dir is None:
        args.output_dir = Path(os.path.abspath(__file__)).parent.parent.parent / "finetune_models"
    if args.metric_path is None:
        args.metric_path = Path(os.path.abspath(__file__)).parent.parent.parent / "hf_metrics"
    if args.dataset is None:
        print("No dataset specified. Exiting")
    if args.output_name is None:
        args.output_name = args.model + "_" + args.dataset
    print("*"*50)
    print(args)

    main(args)
tigerscore/candidates_generation/finetune.py
TIGER-AI-Lab/TIGERScore
import json
import os
import sys
os.environ["TOKENIZERS_PARALLELISM"] = "false"
import torch
import transformers
import logging
import numpy as np
from transformers import (
    TrainingArguments,
    HfArgumentParser,
    Trainer,
    Seq2SeqTrainer,
    DataCollatorForSeq2Seq,
    Seq2SeqTrainingArguments
)
from dataclasses import dataclass, field
from generate_candidates import get_model_size, get_torch_dtype
from typing import Optional, Sequence, Dict, List
from model_utils import (
    build_model,
    build_tokenizer,
)
sys.path.append("..")
from common.evaluation import overall_eval
IGNORE_INDEX = -100

@dataclass
class ModelArguments:
    model_type: str
    model_name_or_path: str
    dtype: str = "float32"
    cache_dir: Optional[str] = None
    

@dataclass
class DataArguments:
    data_dir: str
    train_file: str
    eval_file: str = None
    eval_metrics: List[str] = field(default_factory=lambda: ["bleu", "rouge"])
    input_max_length: int = 512
    output_max_length: int = 128
    with_instruction: bool = False
    
def load_dataset(data_args):
    with open(data_args.train_file, 'r') as f:
        train_data = json.load(f)
    if data_args.eval_file:
        with open(data_args.eval_file, 'r') as f:
            eval_data = json.load(f)
    else:
        eval_data = None

    return train_data, eval_data

class SupervisedDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings

    def __getitem__(self, idx):
        return {key: val[idx] for key, val in self.encodings.items()}

    def __len__(self):
        return len(self.encodings["input_ids"])
    
def preprocess_function(examples, tokenizer, data_args):
    if data_args.with_instruction:
        inputs = [x["instruction"] + "\n" + x["input"] for x in examples]
    else:
        inputs = [x["input"] for x in examples]
    inputs = [x.strip(' \n') for x in inputs]
    outputs = [x["output"] for x in examples]

    logging.warning("# of examples: {}".format(len(inputs)))                
    logging.warning("Example of inputs:")
    print(inputs[0])
    logging.warning("Example of outputs:")
    print(outputs[0])
    
    model_inputs = tokenizer(inputs, max_length=data_args.input_max_length, truncation=True)
    # Setup the tokenizer for targets
    with tokenizer.as_target_tokenizer():
        labels = tokenizer(outputs, max_length=data_args.output_max_length, truncation=True)
    
    logging.warning("Example of model inputs:")
    print("input_ids", model_inputs['input_ids'][0])
    print("attention_mask", model_inputs['attention_mask'][0])
    logging.warning("Example of labels:")
    print(labels['input_ids'][0])
    labels["input_ids"] = [
        [(l if l != tokenizer.pad_token_id else IGNORE_INDEX) for l in label] for label in labels["input_ids"]
    ]
    model_inputs["labels"] = labels["input_ids"]
    return SupervisedDataset(model_inputs)



@dataclass
class DataCollatorForSupervisedDataset(object):
    """Collate examples for supervised fine-tuning."""

    tokenizer: transformers.PreTrainedTokenizer

    def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
        input_ids, labels = tuple([torch.tensor(instance[key]) for instance in instances] for key in ("input_ids", "labels"))
        input_ids = torch.nn.utils.rnn.pad_sequence(
            input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id
        )
        labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=IGNORE_INDEX)
        # print(self.tokenizer.batch_decode(input_ids))
        # print(self.tokenizer.batch_decode(labels.masked_fill(labels == IGNORE_INDEX, self.tokenizer.pad_token_id)))
        # print("##" * 30)
        return dict(
            input_ids=input_ids,
            labels=labels,
            attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
        )


def main(
    model_args: ModelArguments,
    data_args: DataArguments,
    training_args: TrainingArguments,
):
    
    model = build_model(
        model_args.model_type,
        model_args.model_name_or_path,
        torch_dtype=get_torch_dtype(model_args.dtype),
        device_map="auto",
        cache_dir=model_args.cache_dir, resume_download=True)
    n_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
    logging.warning("The {} has {} trainable parameters".format(model_args.model_name_or_path, get_model_size(n_params)))
    tokenizer = build_tokenizer(
        model_args.model_name_or_path,
        cache_dir=model_args.cache_dir, resume_download=True)
    logging.warning("Loading dataset...")

    train_data, eval_data = load_dataset(data_args)
    logging.warning("Dataset loaded.")
    logging.warning("Preprocessing dataset...")
    train_dataset = preprocess_function(train_data, tokenizer, data_args)
    eval_dataset = preprocess_function(eval_data, tokenizer, data_args)
    logging.warning("Dataset preprocessed.")
    logging.warning("Loading data collator...")
    data_collator = DataCollatorForSupervisedDataset(tokenizer)
    logging.warning("Data collator loaded.")
    logging.warning("Loading trainer...")

    def compute_metrics(eval_pred):
        
        logits, labels = eval_pred
        labels[labels == IGNORE_INDEX] = tokenizer.pad_token_id
        logits[logits == IGNORE_INDEX] = tokenizer.pad_token_id
        predictions = tokenizer.batch_decode(logits, skip_special_tokens=True)
        labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
        logging.warning("Example of predictions:")
        print(predictions[:3])
        logging.warning("Example of labels:")
        print(labels[:3])
        scores = overall_eval(predictions, labels, metrics=data_args.eval_metrics)
        return {
            key: np.mean(value) for key, value in scores.items()
        }

    training_args.evaluation_strategy = "epoch"
    training_args.weight_decay=0.01
    training_args.save_total_limit=5
    training_args.predict_with_generate=True
    training_args.generation_num_beams=4
    training_args.generation_max_length=data_args.output_max_length
    training_args.load_best_model_at_end=True
    logging.warning("Training arguments:")
    print(training_args)
    trainer = Seq2SeqTrainer(
        model=model,
        args=training_args,
        tokenizer=tokenizer,
        train_dataset=train_dataset,
        eval_dataset=eval_dataset,
        data_collator=data_collator,
        compute_metrics=compute_metrics,
    )
    logging.warning("Trainer loaded.")
    logging.warning("Training...")
    trainer.train()
    logging.warning("Training finished.")
    logging.warning("Saving model...")
    trainer.save_model(output_dir=os.path.join(training_args.output_dir, "checkpoint-best"))
    logging.warning("Model saved.")
    

if __name__ == "__main__":
    parser = transformers.HfArgumentParser((ModelArguments, DataArguments, Seq2SeqTrainingArguments))
    model_args, data_args, training_args = parser.parse_args_into_dataclasses()
    main(model_args, data_args, training_args)

tigerscore/candidates_generation/finetune_base_model.py
TIGER-AI-Lab/TIGERScore
"""
    This file is modified based on This file is modified based on:
    https://github.com/Ravoxsg/SummaReranker-ACL-22-/blob/main/src/candidate_generation/main_candidate_generation.py
    We thank the authors for sharing their code.
"""
# Generate summary candidates with the fine-tuned models.

import argparse
import sys
import os
import torch
import openai
import json
from tqdm import tqdm
from transformers import AutoProcessor, ViTImageProcessor
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from candidates_generation.engine import (
    beam_search_step,
)
from common.utils import (
    seed_everything,
    str2bool,
    empty2None,
    empty2Noneint,
    fetch_images,
)
from candidates_generation.model_utils import (
    build_model,
    build_tokenizer,
    build_processor,
)
from pathlib import Path

class GenerationDataset(torch.utils.data.Dataset):
    """
        Dataset for generate candidates for given sources
    """

    def __init__(self, tokenizer, data, prompt_max_length, prefix=None, no_instruction=False,model_name=None):
        self.tokenizer = tokenizer
        self.data = data
        self.prompt_max_length = min(prompt_max_length, tokenizer.model_max_length)
        self.prefix = prefix if prefix is not None else ""
        self.no_instruction = no_instruction
        self.model_name = model_name

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        # apply the prompt template to get the proper prompt
        item = self.data[idx]
        if self.no_instruction:
            prompt = item['input']
        else:
            prompt = item['instruction'] + item['input'] # + "\n Description:"
        prompt = self.input2chat(prompt)
        encoded_prompt = self.tokenizer(prompt, max_length=self.prompt_max_length, padding='max_length', truncation=True, return_tensors="pt",return_token_type_ids=False)
        for key in encoded_prompt.keys():
            encoded_prompt[key] = encoded_prompt[key].squeeze(0)
        return {
            "id": item['id'],
            "encodings": encoded_prompt,
            "original": prompt,
        }
        
    def input2chat(self, _input):
        """
            Convert input to chat, For LLAMAs
        """
        
        cov_style = {
            "vicuna": ("A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. ",
                       "USER:",
                       " ASSISTANT:"),
            "Wizard":("Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n",
                      "### Instruction:\n",
                      "\n\n### Response: Let's think step by step."),
            "Llama-2":("[INST] <<SYS>>\nYou are a helpful, respectful and honest assistant.\n<</SYS>>\n\n",
                        "",
                        " [/INST]"),
            "chatglm-6b":("[Round 0]\n",
                          "问：",
                          "\n答："
                ),
            "stablelm":("""<|SYSTEM|># StableLM Tuned (Alpha version)
- StableLM is a helpful and harmless open-source AI language model developed by StabilityAI.
- StableLM is excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.
- StableLM is more than just an information source, StableLM is also able to write poetry, short stories, and make jokes.
- StableLM will refuse to participate in anything that could harm a human.
""",
                        "<|USER|>",
                        "<|ASSISTANT|>"),
        }
        
        sys_prompt = ""
        role_user = ""
        role_assistant = ""
        for name in cov_style.keys():
            if name in self.model_name:
                sys_prompt, role_user, role_assistant = cov_style[name]
                break
        return sys_prompt + role_user + _input + role_assistant
    
class Image2TextGenerationDataset(torch.utils.data.Dataset):
    """
        Dataset for generate candidates for given sources
    """
    def __init__(self, tokenizer, processor, data):
        self.tokenizer = tokenizer
        self.processor = processor
        self.data = data
        
        self.image_urls = [item['input'] for item in data]
        self.images = fetch_images(self.image_urls, 4, retries=2, timeout=10)

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        # apply the prompt template to get the proper prompt
        item = self.data[idx]
        image = self.images[idx]
        pixel_values = self.processor(image, return_tensors="pt").pixel_values
        pixel_values = pixel_values.squeeze(0)
        return {
            "id": item['id'],
            "encodings": {
                "pixel_values": pixel_values
            }
        }

def get_model_size(n_param):
    """
        Get the size of the model in MB
    """
    units = ["K", "M", "B", "T"]
    unit = 0
    while n_param > 1000 and unit < len(units) - 1:
        n_param /= 1000
        unit += 1
    return "{:.2f}{}".format(n_param, units[unit])

def get_torch_dtype(dtype_str):
    """
        Get the torch dtype from a string
    """
    if dtype_str == "float32":
        return torch.float32
    elif dtype_str == "float16":
        return torch.float16
    elif dtype_str == "bfloat16":
        return torch.bfloat16
    elif dtype_str == "int8":
        return torch.int8
    else:
        raise ValueError("Invalid dtype {}".format(dtype_str))
    

def main(args):
    seed_everything(args.seed)
    # device
    device = torch.device("cpu")
    if args.cuda and torch.cuda.is_available():
        device = torch.device("cuda")
    args.device = device
    print("\nUsing device {}".format(device))
    # tokenizer
    tokenizer = build_tokenizer(args.model, cache_dir=args.cache_dir, resume_download=True,trust_remote_code=True)
    # model

    model_kwargs = {
        "torch_dtype": get_torch_dtype(args.dtype),
        "cache_dir": args.cache_dir,
        "resume_download": True,
        "trust_remote_code":True,
        "device_map":"auto"
    }
    if "7b" in args.model:
        model_kwargs.pop("device_map")
        
    model = build_model(
        args.model_type,
        args.model,
        **model_kwargs
    )
    if "device_map" not in model_kwargs:
        model = model.to(device)
    # model = model.to(device)
    n_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
    print("\nThe {} has {} trainable parameters".format(args.model, get_model_size(n_params)))
    
    datasets = args.dataset.split(',')
    sets = args.set.split(',')
    for dataset_name in datasets:
        for set in sets:
            print("\nGenerating candidates for {}-{}".format(dataset_name, set))
            
            data_file = Path(args.data_dir) / dataset_name.replace(":", "/") / f"{set}_data.json"
            save_file = Path(args.data_dir) / dataset_name.replace(":", "/") / "candidates" / set / args.decoding_method / f"{args.model.split('/')[-1]}.json"
            # data
            data = json.load(open(data_file, 'r'))
            if args.end_idx is not None:
                data = data[:args.end_idx]
            if args.start_idx is not None:
                data = data[args.start_idx:]
            
            if isinstance(args.max_size, int) and args.max_size > 0:
                print("Truncating data from {} to {}".format(len(data), args.max_size))
                data = data[:args.max_size]
            if len(data) == 0:
                print("No data to generate")
                return

            if args.start_idx is not None or args.end_idx is not None:
                if args.start_idx is None:
                    file_postfix = "0-{}".format(args.end_idx)
                elif args.end_idx is None:
                    file_postfix = "{}-{}".format(args.start_idx, len(data)+args.start_idx)
                else:
                    file_postfix = "{}-{}".format(args.start_idx, args.end_idx)
                save_file = save_file.parent / "{}_{}.json".format(save_file.stem, file_postfix)
            save_file.parent.mkdir(parents=True, exist_ok=True)
            # check if the data have already been generated
            if os.path.exists(save_file):
                print("Found existing candidates.")
                if args.overwrite:
                    print("Overwriting existing data")
                else:
                    print("Not overwriting existing data. Finishing generating")
                    continue
            else:
                print("No existing candidates found. Generating candidates")

            if not args.image2text:
                dataset = GenerationDataset(tokenizer, data, args.prompt_max_length, no_instruction=args.no_instruction, model_name=args.model)
            else:
                # image processor
                processor = build_processor(args.model_type, args.model, cache_dir=args.cache_dir)
                dataset = Image2TextGenerationDataset(tokenizer, processor, data)
            print("Total size of dataset: {}".format(len(dataset)))
            # data loader
            dataloader = torch.utils.data.DataLoader(dataset, batch_size = args.inference_bs, shuffle = False)

            # summary generation
            candidates = []
            with torch.no_grad():
                for idx, batch in tqdm(enumerate(dataloader), total = len(dataloader), desc = "Generating candidates"):
                    for k in batch['encodings'].keys():
                        batch['encodings'][k] = batch['encodings'][k].to(device)
                    # generate candidates
                    outputs = beam_search_step(
                        inputs = batch['encodings'],
                        tokenizer=tokenizer,
                        base_model=model,
                        args=args,
                        # pad_token_id=tokenizer.pad_token_id, # debug for alpaca
                    )
                    _candidates = outputs['generated']
                    _logprobs = outputs['logprobs']
                    for id, _c, _l in zip(batch['id'], _candidates, _logprobs):
                        candidates.append({
                            "id": id,
                            "candidates": [
                                {
                                    "text": _c[i].strip(' \n'),
                                    "scores": {
                                        "logprobs": _l[i]
                                    }
                                } 
                                for i in range(len(_c))
                            ]
                        })
            
            print("Total # of candidates: {}".format(len(candidates)))
            print("# of candidates per example: {}".format(len(list(candidates[0].values())[0])))
            # save 
            json.dump(candidates, open(save_file, 'w'), indent=4, ensure_ascii=False)
            print("Saved candidates to {}".format(save_file))
    
    print("Done generating candidates!")

if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument('--seed', type = int, default = 42)
    parser.add_argument('--cuda', type = str2bool, default = True)

    # data
    parser.add_argument('--data_dir', type = str, default = '../../data')
    parser.add_argument('--dataset', type = empty2None, required=True)
    parser.add_argument('--set', type = str, default = "test")
    parser.add_argument('--max_size', type = int, default = None)

    # model
    parser.add_argument('--model_type', type = str, default = "flan-t5",)
    parser.add_argument('--model', type = str, default = "google/flan-t5-xxl")
    parser.add_argument('--dtype', type = str, default = "float32",
                        choices = ["float32", "float16", "bfloat16", "int8"])
    parser.add_argument('--cache_dir', type = str, default = None)
    parser.add_argument('--image2text', type = str2bool, default = False)

    # candidate generation
    parser.add_argument('--inference_bs', type = int, default = 2)
    parser.add_argument('--decoding_method', type = str, default = "diverse_beam_search",
                        choices = ["beam_search", "diverse_beam_search", "top_p_sampling", "top_k_sampling"])
    parser.add_argument('--num_return_sequences', type = int, default = 1) # default: 1
    parser.add_argument('--num_beams', type = int, default = 1) # for beam search
    parser.add_argument('--num_beam_groups', type = int, default = 1) # for diverse beam search
    parser.add_argument('--diversity_penalty', type = float, default = 1.0) # for diverse beam search
    parser.add_argument('--top_p', type = float, default = 1.0) # for top-p sampling
    parser.add_argument('--top_k', type = int, default = 50) # for top-k sampling
    parser.add_argument('--temperature', type = float, default = 0.8) # for top-p and top-k sampling
    parser.add_argument('--stemmer', type = str2bool, default = True)

    # generation config
    parser.add_argument('--prompt_max_length', type = int, default = 512)
    parser.add_argument('--output_max_length', type = int, default = 512)
    parser.add_argument('--length_penalty', type = float, default = 1.0)
    parser.add_argument('--repetition_penalty', type = float, default = 1.0)
    parser.add_argument('--no_repeat_ngram_size', type = int, default = 0)
    parser.add_argument('--no_instruction', type = str2bool, default = False)
    
    parser.add_argument('--start_idx', type = empty2Noneint, default = None)
    parser.add_argument('--end_idx', type = empty2Noneint, default = None)

    parser.add_argument('--overwrite', type = str2bool, default = True)
    args = parser.parse_args()

    if args.cache_dir is None:
        args.cache_dir = Path(os.path.abspath(__file__)).parent.parent.parent / "hf_models"
    if args.dataset is None:
        print("No dataset specified. Exiting")
    print("*"*50)
    print(args)

    main(args)





tigerscore/candidates_generation/generate_candidates.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_wmt.py
"""
import json
import random
import logging
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem
logging.basicConfig(level=logging.warning)

def main(
    task: str,
    data_path: str,
    dataset: str,
    output_file: str = None,
    xgptscore_mode: str = "instruction",
    model_name: str = "ChatGPT",
    overwrite: bool = False,
    max_size: int = None,
    seed: int = 42,
    shuffle_file: bool = False,
    source_max_length: int = None,
    ref_max_length: int = None,
    hypo_max_length: int = None,
    dataset_split: str = "test",
):
    logging.warning("Params: \n{}".format(json.dumps(locals(), indent=4)))
    # load data
    data_path = Path(data_path)
    input_file = data_path / dataset / (dataset_split + "_data.json")
    
    input_file=Path(input_file)
    if not output_file:
        output_file = data_path / dataset / "candidates" / dataset_split / "top_p_sampling" / f"{model_name}.json"
        if not output_file.parent.parent.exists():
            output_file.parent.parent.mkdir(parents=True)
        if not output_file.parent.exists():
            output_file.parent.mkdir()
    else:
        output_file = Path(output_file)
    with open(input_file, "r") as f:
        items = json.load(f)
        logging.warning("Loaded {} items from {}".format(len(items), input_file))
    logging.warning("Preparing writing to {}...".format(output_file))
    
    random.seed(seed); logging.warning("Set seed to {}".format(seed))
    if shuffle_file:
        random.shuffle(items)
        logging.warning("Shuffled {} items".format(len(items)))
    if isinstance(max_size, int) and max_size > 0:
        items = items[:max_size]
        logging.warning("Truncated to {} items".format(len(items)))

    xgptitems = []
    for item in items:
        xgptitems.append(XPGTItem(
            task=task,
            instruction=item['instruction'],
            input=item['input'],
            ref_output=item['output'] if "output" in item else item['refs'],
            hypo_output=None,
        ))
        if "candidates" in item:
            del item["candidates"]

    if not output_file.exists() or overwrite:
        logging.warning("Running xgptscore")
        # run xgptscore
        xgptscore_params = {
            "max_lengths": {
                "input": source_max_length,
                "hypo_output": hypo_max_length,
                "ref_output": ref_max_length,
            },
        }
        result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name, **xgptscore_params)
        for i, item in enumerate(items):
            item['responses'] = result['round_completions'][i]
            item['messages_records'] = result['messages_records'][i]
            item['candidates'] = [
                {"text":result['round_completions'][i][0],
                               "scores":{}
            }]
        # print(items)
        with open(output_file, "w") as f:
            json.dump(items, f, indent=4, ensure_ascii=False)
            logging.warning("Saved to {}".format(output_file))
    else:
        logging.warning("Loading from {}".format(output_file))
        with open(output_file, "r") as f:
            items = json.load(f)

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--task", type=str)
    parser.add_argument("--data_path", type=str)
    parser.add_argument("--dataset", type=str)
    args = parser.parse_args()
    main(**vars(args))

tigerscore/candidates_generation/generate_candidates_by_gpt.py
TIGER-AI-Lab/TIGERScore
from transformers import (
    AutoTokenizer, 
    AutoModelForSeq2SeqLM,
    AutoModelForCausalLM,
    AutoModel,
    VisionEncoderDecoderModel,
    ViTImageProcessor,
)
decoder_only_models = ["alpaca", "llama", "opt", "bloom", "gpt", "vicuna","koala","Wizard","stablelm"]
def build_model(model_type, model_name, **kwargs):
    """
        Build the model from the model name
    """
    if any([x in model_type for x in decoder_only_models]) or any([x in model_name for x in decoder_only_models]):
        model = AutoModelForCausalLM.from_pretrained(model_name, **kwargs)
    elif model_type in ["vit"]:
        model = VisionEncoderDecoderModel.from_pretrained(model_name)
    elif model_type in ["bart", "t5", "mbart", "m2m100", "nllb", "opus_mt", "unifiedqa", "opus-mt", "pegasus"]:
        model = AutoModelForSeq2SeqLM.from_pretrained(model_name, **kwargs)
    else:
        model = AutoModel.from_pretrained(model_name, **kwargs)

    return model

def build_tokenizer(model_name, **kwargs):
    """
        Build the tokenizer from the model name
    """

    if "vicuna" in model_name:
        tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left", use_fast=False, **kwargs)
    # elif "Wizard" in model_name:
    #     tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left", return_token_type_ids=False, **kwargs)
    elif any([x in model_name for x in decoder_only_models]):
        # padding left
        tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left", **kwargs)
    else:
        tokenizer = AutoTokenizer.from_pretrained(model_name, **kwargs)#, use_fast=False)
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
        tokenizer.pad_token_id = tokenizer.eos_token_id
    return tokenizer

def build_processor(model_type, model_name, **kwargs):
    """
        Build the processor from the model name
    """
    if model_type in ["vit"]:
        processor = ViTImageProcessor.from_pretrained(model_name, **kwargs)
    else:
        raise NotImplementedError
    return processor
tigerscore/candidates_generation/model_utils.py
TIGER-AI-Lab/TIGERScore
import torch
from typing import Dict, TypeVar, Iterable, List
import transformers
from transformers import LlamaForCausalLM, LlamaTokenizer
from tqdm import tqdm

T = TypeVar('T')

DEFAULT_PAD_TOKEN = "[PAD]"
DEFAULT_EOS_TOKEN = "</s>"
DEFAULT_BOS_TOKEN = "</s>"
DEFAULT_UNK_TOKEN = "</s>"


def smart_tokenizer_and_embedding_resize(
    special_tokens_dict: Dict,
    tokenizer: transformers.PreTrainedTokenizer,
):
    """Resize tokenizer and embedding.
    Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
    """
    tokenizer.add_special_tokens(special_tokens_dict)
    tokenizer.add_special_tokens(
        {
            "eos_token": DEFAULT_EOS_TOKEN,
            "bos_token": DEFAULT_BOS_TOKEN,
            "unk_token": DEFAULT_UNK_TOKEN,
        }
    )


class InstructScore:
    def __init__(self, batch_size=2, max_src_len=512, max_trg_len=512, device_id="cuda"):
        self.batch_size = batch_size
        self.max_src_len = max_src_len
        self.max_trg_len = max_trg_len
        self.device_id = device_id
        print("Max source length: ", max_src_len)
        print("MAX target length: ", max_trg_len)

        print("Loading InstructScore model and tokenizer... ")
        self.tokenizer = LlamaTokenizer.from_pretrained(
            "xu1998hz/InstructScore", model_max_length=max_src_len, use_fast=False
        )
        # enable batch inference by left padding
        self.tokenizer.padding_side = "left"

        smart_tokenizer_and_embedding_resize(
            special_tokens_dict=dict(pad_token=DEFAULT_PAD_TOKEN),
            tokenizer=self.tokenizer,
        )
        self.model = LlamaForCausalLM.from_pretrained("xu1998hz/InstructScore").to(
            device_id
        )
        self.model.eval()

    def score(self, ref_ls, out_ls):
        assert len(ref_ls) == len(out_ls), "The number of references and outputs should be the same."
        if len(ref_ls) == 0 or len(out_ls) == 0:
            return [], []

        if isinstance(ref_ls, str):
            ref_ls = [ref_ls]
        if isinstance(out_ls, str):
            out_ls = [out_ls]

        prompt_ls = [
            f'You are evaluating Chinese-to-English Machine translation task. The correct translation is "{ref}". The model generated translation is "{out}". Please identify all errors within each model output, up to a maximum of five. For each error, please give me the corresponding error type, major/minor label, error location of the model generated translation and explanation for the error. Major errors can confuse or mislead the reader due to significant change in meaning, while minor\
         errors don\'t lead to loss of meaning but will be noticed.'
            for ref, out in zip(ref_ls, out_ls)
        ]

        with torch.no_grad():
            batch_outputs_all = []
            scores_ls_all = []
            for prompt_batch in tqdm(batchify(prompt_ls, self.batch_size)):
                inputs = self.tokenizer(
                    prompt_batch,
                    return_tensors="pt",
                    padding=True,
                    truncation=True,
                    max_length=self.max_src_len,
                )
                outputs = self.model.generate(
                    inputs["input_ids"].to(self.device_id),
                    attention_mask=inputs["attention_mask"].to(self.device_id),
                    max_new_tokens=self.max_trg_len,
                )
                batch_outputs = self.tokenizer.batch_decode(
                    outputs,
                    skip_special_tokens=True,
                    clean_up_tokenization_spaces=True,
                )
                scores_ls = [
                    (-1) * output.count("Major/minor: Minor")
                    + (-5) * output.count("Major/minor: Major")
                    for output in batch_outputs
                ]
                batch_outputs_all.extend(batch_outputs)
                scores_ls_all.extend(scores_ls)
            return batch_outputs_all, scores_ls_all


def batchify(data: Iterable[T], batch_size: int) -> Iterable[List[T]]:
    assert batch_size > 0

    batch = []
    for item in data:
        # Yield next batch
        if len(batch) == batch_size:
            yield batch
            batch = []

        batch.append(item)

    # Yield last un-filled batch
    if len(batch) != 0:
        yield batch  


def main():
    device_id = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

    refs = [
        "SEScore is a simple but effective next generation text generation evaluation metric",
        "SEScore it really works",
    ]
    outs = [
        "SEScore is a simple effective text evaluation metric for next generation",
        "SEScore is not working",
    ]

    scorer = InstructScore(device_id=device_id)
    batch_outputs, scores_ls = scorer.score(refs, outs)
    print(batch_outputs)
    print(scores_ls)


if __name__ == "__main__":
    main()
tigerscore/common/InstructScore.py
TIGER-AI-Lab/TIGERScore
import os
import sys
cur_folder= os.path.dirname(os.path.abspath(__file__))
if cur_folder not in sys.path:
    sys.path.append(cur_folder)

tigerscore/common/__init__.py
TIGER-AI-Lab/TIGERScore
# %%
import torch
import torch.nn as nn
import traceback
from transformers import BartTokenizer, BartForConditionalGeneration


class BARTScorer:
    def __init__(self, device='cuda:0', max_length=1024, checkpoint='facebook/bart-large-cnn'):
        # Set up model
        self.device = device
        self.max_length = max_length
        self.tokenizer = BartTokenizer.from_pretrained(checkpoint)
        self.model = BartForConditionalGeneration.from_pretrained(checkpoint)
        self.model.eval()
        self.model.to(device)

        # Set up loss
        self.loss_fct = nn.NLLLoss(reduction='none', ignore_index=self.model.config.pad_token_id)
        self.lsm = nn.LogSoftmax(dim=1)

    def load(self, path='./models/bart.pth'):
        """ Load model from paraphrase finetuning """
        self.model.load_state_dict(torch.load(path, map_location=self.device))

    def score(self, srcs, tgts, batch_size):
        """ Score a batch of examples """
        score_list = []
        for i in range(0, len(srcs), batch_size):
            src_list = srcs[i: i + batch_size]
            tgt_list = tgts[i: i + batch_size]
            try:
                with torch.no_grad():
                    encoded_src = self.tokenizer(
                        src_list,
                        max_length=self.max_length,
                        truncation=True,
                        padding=True,
                        return_tensors='pt'
                    )
                    encoded_tgt = self.tokenizer(
                        tgt_list,
                        max_length=self.max_length,
                        truncation=True,
                        padding=True,
                        return_tensors='pt'
                    )
                    src_tokens = encoded_src['input_ids'].to(self.device)
                    src_mask = encoded_src['attention_mask'].to(self.device)

                    tgt_tokens = encoded_tgt['input_ids'].to(self.device)
                    tgt_mask = encoded_tgt['attention_mask']
                    tgt_len = tgt_mask.sum(dim=1).to(self.device)

                    output = self.model(
                        input_ids=src_tokens,
                        attention_mask=src_mask,
                        labels=tgt_tokens
                    )
                    logits = output.logits.view(-1, self.model.config.vocab_size)
                    loss = self.loss_fct(self.lsm(logits), tgt_tokens.view(-1))
                    loss = loss.view(tgt_tokens.shape[0], -1)
                    loss = loss.sum(dim=1) / tgt_len
                    curr_score_list = [-x.item() for x in loss]
                    score_list += curr_score_list

            except RuntimeError:
                traceback.print_exc()
                print(f'source: {src_list}')
                print(f'target: {tgt_list}')
                exit(0)
        return score_list

tigerscore/common/bart_score.py
TIGER-AI-Lab/TIGERScore
import numpy as np
import scipy
def cor_pearson(hypo_scores, ref_scores):
    """
    Args:
        hypo_scores: ndarray of shape (n, c) where n is the number of samples, c is the number of candidates
        ref_scores: ndarray of shape (n, c) where n is the number of samples, c is the number of candidates
    returns:
        cor: float, the mean correlation coefficient
    """
    if isinstance(hypo_scores, list):
        hypo_scores = np.array(hypo_scores)
    if isinstance(ref_scores, list):
        ref_scores = np.array(ref_scores)
    assert hypo_scores.shape == ref_scores.shape
    bz, c = hypo_scores.shape
    hypo_scores = hypo_scores.reshape(bz, c).T
    ref_scores = ref_scores.reshape(bz, c).T
    cor = 0
    for i in range(c):
        cor += np.corrcoef(hypo_scores[i], ref_scores[i])[0, 1]
    cor /= c
    return cor

def cor_spearman(hypo_scores, ref_scores):
    """
    Args:
        hypo_scores: ndarray of shape (n, c) where n is the number of samples, c is the number of candidates
        ref_scores: ndarray of shape (n, c) where n is the number of samples, c is the number of candidates
    returns:
        cor: float, the mean of the diagonal elements of the spearman correlation matrix
    """
    if isinstance(hypo_scores, list):
        hypo_scores = np.array(hypo_scores)
    if isinstance(ref_scores, list):
        ref_scores = np.array(ref_scores)
    assert hypo_scores.shape == ref_scores.shape
    bz, c = hypo_scores.shape
    hypo_scores = hypo_scores.reshape(bz, c).T
    ref_scores = ref_scores.reshape(bz, c).T
    cor = 0
    for i in range(c):
        cor += scipy.stats.spearmanr(hypo_scores[i], ref_scores[i]).correlation
    cor /= c
    return cor

            
def cor_spearman_footrule(hypo_scores, ref_scores):
    """
    Args:
        hypo_scores: ndarray of shape (n, c) where n is the number of samples, c is the number of candidates
        ref_scores: ndarray of shape (n, c) where n is the number of samples, c is the number of candidates
    returns:
        cor: float, the mean of the set of the spearman correlation coefficients
    """
    if isinstance(hypo_scores, list):
        hypo_scores = np.array(hypo_scores)
    if isinstance(ref_scores, list):
        ref_scores = np.array(ref_scores)
    assert hypo_scores.shape == ref_scores.shape
    bz, c = hypo_scores.shape
    hypo_scores = hypo_scores.reshape(bz, c)
    ref_scores = ref_scores.reshape(bz, c)
    return np.abs(hypo_scores - ref_scores).sum(axis=-1).mean()
tigerscore/common/cor_eval.py
TIGER-AI-Lab/TIGERScore
# Set dataset configurations here
# the slices of each split should be all positive integers
NUM_TRAIN_EXAMPLES = 5000 
NUM_VAL_EXAMPLES = 200
NUM_TEST_EXAMPLES = 200 # human evaluation 1000 * 5 = 5000 > 1000/500
DATASETS_CONFIG = {
    "summarization": {
        "cnn_dailymail:3.0.0": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
                "test": f"test[:{NUM_TEST_EXAMPLES}]",
            },
            "input_key": "article",
            "output_key": "highlights",
            "instruction": "Summarize the news article.",
        },
        "xsum": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
                "test": f"test[:{NUM_TEST_EXAMPLES}]",
            },
            "input_key": "document",
            "output_key": "summary",
            "instruction": "Generate an abstractive summary for the following article.",
        },
        "gigaword": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
                "test": f"test[:{NUM_TEST_EXAMPLES}]",
            },
            "input_key": "document",
            "output_key": "summary",
            "instruction": "Generate a title for this article.",
        },
        "samsum": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
                "test": f"test[:{NUM_TEST_EXAMPLES}]",
            },
            "input_key": "dialogue",
            "output_key": "summary",
            "instruction": "Generate a summary for the following dialogue.",
        },
    },
    "translation": {
        "wmt19:zh-en": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
            },
            "input_key": "zh",
            "output_key": "en",
            "instruction": "Translate the following Chinese text into English.",
        },
        "wmt19:de-en": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
            },
            "input_key": "en",
            "output_key": "de",
            "instruction": "Translate the following English text into German.",
        },
        "wmt19:cs-en": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
            },
            "input_key": "cs",
            "output_key": "en",
            "instruction": "Translate the following Czech text into English.",
        },
        "wmt19:ru-en": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
            },
            "input_key": "en",
            "output_key": "ru",
            "instruction": "Translate the following English text into Russian.",
        },
        "wmt19:fr-de": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
            },
            "input_key": "fr",
            "output_key": "de",
            "instruction": "Translate the following French text into German.",
        }
    },
    "data2text": {
        # Table 2 text, facts
        # "dart":{
        #     "split_info": {
        #         "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
        #         "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
        #         "test": f"test[:{NUM_TEST_EXAMPLES}]",
        #     },
        #     "input_key": "tripleset",
        #     "output_key": "annotations",
        #     "instruction": "Generate a description for the following tuples.",
        # },
        # "totto":{
        #     "split_info": {
        #         "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
        #         "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
        #         "test": f"test[:{NUM_TEST_EXAMPLES}]",
        #     },
        #     "input_key": "tripleset",
        #     # The input is complex, see the python file for more details
        #     "output_key": "sentence_annotations",
        #     "instruction": "Generate a description for the following table.",
        # },
        # "kasnerz/wikitabletext":{
        #     "split_info": {
        #         "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
        #         "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
        #         "test": f"test[:{NUM_TEST_EXAMPLES}]",
        #     },
        #     "input_key": "table",
        #     # The input is complex, see the python file for more details
        #     "output_key": "reference",
        #     "instruction": "Generate a description for the following table.",            
        # },
        "web_nlg:release_v2.1":{
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"dev[:{NUM_VAL_EXAMPLES}]",
                "test": f"test[:{NUM_TEST_EXAMPLES}]",
            },
            "input_key": "modified_triple_sets",
            # The input is complex, see the python file for more details
            "output_key": "lex",
            "instruction": "Generate a description for the following table.",  
        }
    },
    "long-form QA": {
        "din0s/asqa": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"dev[:{NUM_VAL_EXAMPLES}]",
                "test": f"dev[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "input_key": "ambiguous_question",
            "output_key": "annotations",
            "instruction": "Answer the following ambiguous factoid question by introducing additional knowledge, clarifying the relationship between multiple possible answers (if any) and resolving the ambiguity"
        }, # use llm to generate the answer
        "DongfuTingle/FeTaQA": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
                "test": f"test[:{NUM_TEST_EXAMPLES}]",
            },
            # special format
            "input_key": None,
            "output_key": "answer",
            "id_key": "feta_id", 
            "instruction": "Given a table title and some highlighed cells (rows), answer the following question in several sentences with proper reasoning.",
        }, # using llm to generate the answer
        "cosmos_qa": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES//2}]",
                "finetune": f"train[{NUM_TRAIN_EXAMPLES//2}:]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
                "test": f"validation[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "input_key": None,
            "output_key": None,
            "instruction": "Given the context below, answer the following question in 1 or 2 sentences.",
        }, # try fine-tuning on flan-t5 or using llm directly
        "eli5":{
            "split_info": {
                "train": f"train_eli5[:{NUM_TRAIN_EXAMPLES}]",
                "finetune": f"train_eli5[{NUM_TRAIN_EXAMPLES}:]",
                "validation": f"validation_eli5[:{NUM_VAL_EXAMPLES}]",
                "test": f"test_eli5[:{NUM_TEST_EXAMPLES}]",
            },
            "input_key": 'title',
            "output_key": 'answers',
            "instruction": "Answer the following question step by step in multiple sentences.",
        } # try fine-tuning on flan-t5 or using llm directly
    },
    "mathQA": {
        "gsm8k:main": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"test[:{NUM_VAL_EXAMPLES}]",
                "test": f"test[:]",
            },
            "input_key": "question",
            "output_key": "answer",
            "instruction": "Answer the following math word problem step-by-step.",
            # "instruction": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n",
        }, # use dslack/GSM8K-Flan-T5-Large
        # "math_qa": {
        #     "split_info": {
        #         "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
        #         "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
        #         "test": f"test[:{NUM_TEST_EXAMPLES}]",
        #     },
        #     "input_key": "Problem",
        #     "output_key": "Rationale",
        #     # "instruction": "Please, answer the following math problem step-by-step:",
        #     # "instruction": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n",
        #     "instruction": "Solve the following multiple choice math word problem step-by-step.",
        # }, # train our own, rationale 
        # "qwedsacf/grade-school-math-instructions": {  # It's gsm8k with instructions
        #     "split_info": {
        #         "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
        #         "validation": f"train[:{NUM_VAL_EXAMPLES}]",
        #         "test": f"train[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
        #     }, # This one is special, only contains about 8k examples
        #     "input_key": "INSTRUCTION", 
        #     "output_key": "RESPONSE",
        #     "instruction": "Please, answer the following math problem step-by-step:",
        # }, # use jordiclive/alpaca_gpt4-dolly_15k-vicuna-lora-7b. 
        # "competition_math": { # TOO HARD
        #     "split_info": {
        #         "train": f"train[:{NUM_TRAIN_EXAMPLES}]", 
        #         "validation": f"test[:{NUM_VAL_EXAMPLES}]",
        #         "test": f"test[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
        #     },
        #     "input_key": "problem",
        #     "output_key": "solution",
        #     "instruction": "Please, answer the following math problem step-by-step:",
        # }, # try chavinlo/alpaca-native
    },
    "code": {
        "deepmind/code_contests": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"test[:{NUM_VAL_EXAMPLES}]",
                "test": f"test[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "input_key": "description",
            "output_key": "solutions",
            "instruction": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:",
        }, # use dslack/GSM8K-Flan-T5-Large
    },
    "instruction-following": {
        "GAIR/lima": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"test[:]",
                "test": f"test[:]",
            },
            "input_key": "conversations",
            "output_key": None,
            "instruction": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.",
        }, 
        "tatsu-lab/alpaca_farm:alpaca_instructions": {
            "split_info": {
                "train": f"sft[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"val[:{NUM_VAL_EXAMPLES}]",
            },
            "input_key": "instruction+input", # special format
            "output_key": "output",
            "instruction": None, # fill in the dataset instruction
        },
        "HuggingFaceH4/oasst1_en": {
            "split_info": {
                "train": f"train_ift[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"test_ift[:{NUM_VAL_EXAMPLES}]",
                "test": f"test_ift[:{NUM_TEST_EXAMPLES}]",
            },
            "input_key": "messages", # special format
            "output_key": None,
            "instruction": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.",
        },
        "JosephusCheung/GuanacoDataset": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "test": f"train[{NUM_TRAIN_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "input_key": "instruction+input", # special format
            "output_key": "output",
            "instruction": None, # fill in the dataset instruction
        },
        "databricks/databricks-dolly-15k": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "test": f"train[{NUM_TRAIN_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "input_key": "instruction+input", # special format
            "output_key": "output",
            "instruction": None, # fill in the dataset instruction
        }
    },
    "story_generation": {
        "wza/roc_stories": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"train[{NUM_TRAIN_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES}]",
                "test": f"train[{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "input_key": None,
            "output_key": None,
            "instruction": "Generate a reasonable ending for the following story.",
        }, # special format
        "hellaswag": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
                "test": f"validation[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]", # hellaswag did not provide the label for the test set
            },
            "input_key": None,
            "output_key": None,
            "instruction": "Generate a reasonable ending for the following story.",
            # startphrase is the first sentence. 4
        }, # special format. try flan-t5. it has been trained on hellaswag in a mutli-option setting. We could try if it works in the generation setting.
        # "swag": {
        #     "split_info": {
        #         "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
        #         "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
        #         "test": f"validation[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
        #     },
        #     "input_key": None,
        #     "output_key": None,
        #     "instruction": "Generate a reasonable ending for the following story.",
        # }, # special format
    }, # I have a feel that these story generation task is too easy and the reference are not long answers. We can consider removing them.
    
    "other": { # Need to fill
        "common_gen": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
                "test": f"validation[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "input_key": "concepts",
            "output_key": "target",
            "instruction": "Generate a reasonable sentence using the following concepts.",
        }, # special format, use flan-t5. 
        # "lighteval/lsat_qa:all": {
        #     "split_info": {
        #         "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
        #         "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
        #         "test": f"validation[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
        #     },
        #     "input_key": "concepts",
        #     "output_key": None,
        #     "instruction": "Answer the following question given the context carefully, in detail, in more than 50 words.",
        # },
        "vicgalle/alpaca-gpt4": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"train[{NUM_TRAIN_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES}]",
                "test": f"train[{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "input_key": "concepts",
            "output_key": "output",
            "instruction": "",
        },
        "xnli:en": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
                "test": f"validation[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "input_key": "premise",
            "output_key": "hypothesis",
            "instruction": "Generate a reasonable hypothesis given the premise.",
        },
        "knkarthick/dialogsum": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"validation[:{NUM_VAL_EXAMPLES}]",
                "test": f"validation[{NUM_VAL_EXAMPLES}:{NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "input_key": "dialogue",
            "output_key": "summary",
            "instruction": "Summarize the dialogue.",
        },
    }, # do not use
    "image_captioning": {
        "conceptual_captions:labeled": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"train[{NUM_TRAIN_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES}]",
                "test": f"train[{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "id_key": "image_url",
            "input_key": "image_url",
            "output_key": "caption",
            "instruction": ""
        },
        "facebook/winoground": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"train[{NUM_TRAIN_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES}]",
                "test": f"train[{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "id_key": None,
            "input_key": "image_0",
            "output_key": "caption_0",
            "instruction": "",
        },
        "ChristophSchuhmann/MS_COCO_2017_URL_TEXT": {
            "split_info": {
                "train": f"train[:{NUM_TRAIN_EXAMPLES}]",
                "validation": f"train[{NUM_TRAIN_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES}]",
                "test": f"train[{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES}:{NUM_TRAIN_EXAMPLES+NUM_VAL_EXAMPLES+NUM_TEST_EXAMPLES}]",
            },
            "id_key": None,
            "input_key": "URL",
            "output_key": "TEXT",
            "instruction": "",
        }
    }, # do not use
}
tigerscore/common/datasets_config.py
TIGER-AI-Lab/TIGERScore

import psutil
import os
import numpy as np
import spacy
import torch
from copy import deepcopy
from evaluate import load
from sacrebleu import sentence_bleu, corpus_bleu
from nltk import word_tokenize
from typing import List, Optional, Union, Dict, Tuple
from absl import logging
from torch import split
from tqdm import tqdm
from nltk import sent_tokenize
from rouge_score import rouge_scorer
from tqdm.contrib.concurrent import process_map
from pycocoevalcap.spice.spice import Spice
from pycocoevalcap.cider.cider import Cider
from pycocoevalcap.bleu.bleu import Bleu
from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer
from scipy.stats import pearsonr, spearmanr
from collections import defaultdict

logging.set_verbosity(logging.WARNING)


SUPPORTED_METRICS = ['rouge1', 'rouge2', 'rougeL', 'rougeLsum', 'bleu', 'bleurt', "cider", "spice", "bleu4", "bertscore","prism","comet","bart_score","bart_score_cnn","bart_score_para","chrf"]
METRIC_WEIGHTS = {
    "rouge1": 1.0,
    "rouge2": 1.0,
    "rougeL": 1.0,
    "rougeLsum": 1.0,
    "bleu": 0.01,
    "bleu4": 0.01,
    "bleurt": 1.0,
    "cider": 0.01,
    "spice": 0.01,
    "bertscore": 1.0,
    "chrf": 1.0,
    "prism": 1.0,
    "comet_da": 0.01,
    "cometkiwi_da": 0.01,
    "unieval_sum": 0.01,
    "unieval_dialogue": 0.01,
    "unieval_data2text": 0.01,
    "unieval_fact": 0.01,
    "novel_unigram": 1.0,
    "bart_score": 1.0,
    "bart_score_cnn": 1.0,
    "bart_score_para": 1.0,
} # scale to 0-1

def pre_rouge_processing(summary):
    summary = summary.replace("<n>", " ")
    summary = "\n".join(sent_tokenize(summary))
    return summary

def eval_rouge(
    hypotheses: List[List[str]],
    references: List[List[str]],
    rouge_types: List[str]=['rouge1', 'rouge2', 'rougeL', 'rougeLsum']
    ) -> Dict[str, float]:
    """
    Evaluate the hypothesis and reference using the metric.

    Args:
        hypotheses: the hypotheses
        references: the references
        rouge_types: the rouge types to be used.

    Returns:
        A dict of rouge scores.
        key is the rouge type, value is the rouge score, in same shape with hypotheses.
    """
    assert len(hypotheses) == len(references)
    assert set(rouge_types) <= set(['rouge1', 'rouge2', 'rougeL', 'rougeLsum']), "Rouge types should be in ['rouge1', 'rouge2', 'rougeL', 'rougeLsum']"
    scorer = rouge_scorer.RougeScorer(rouge_types, use_stemmer=True, split_summaries=True)
    rouge_scores = {rouge_type: [[] for _ in range(len(hypotheses))] for rouge_type in rouge_types}
    with tqdm(total=len(hypotheses), desc="Evaluating rouge") as pbar:
        for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
            for hypo in hypo_group:
                scores = scorer.score_multi(ref, pre_rouge_processing(hypo))
                for rouge_type in rouge_types:
                    rouge_scores[rouge_type][i].append(scores.get(rouge_type).fmeasure)
            pbar.update(1)
    return rouge_scores

def eval_bleu(
    hypotheses: List[List[str]],
    references: List[List[str]],
    ) -> List[float]:
    """
    Evaluate the hypothesis and reference using the metric.

    Args:
        hypotheses: the hypotheses
        references: the references

    Returns:
        A list of bleu scores, in same shape with hypotheses.
    """
    assert len(hypotheses) == len(references), f"Length of hypotheses {len(hypotheses)} and references {len(references)} should be the same."
    bleu_scores = []
    with tqdm(total=len(hypotheses), desc="Evaluating bleu") as pbar:
        for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
            bleu_scores.append([])
            for hypo in hypo_group:
                bleu_scores[i].append(sentence_bleu(hypo, ref).score)
            pbar.update(1)
    return bleu_scores

def eval_bleurt(
    hypotheses: List[List[str]],
    references: List[List[str]]
    ) -> List[float]:
    """
    Evaluate the hypothesis and reference using the metric.

    Args:
        hypotheses: the hypotheses
        references: the references
    """
    assert len(hypotheses) == len(references)
    bleurt_scorer = load('bleurt')
    bleurt_scores = []
    with tqdm(total=len(hypotheses), desc="Evaluating bleurt") as pbar:
        for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
            bleurt_scores.append([])
            for hypo in hypo_group:
                _t_scores = []
                for _ref in ref:
                    result = bleurt_scorer.compute(predictions=[hypo], references=[_ref])
                    _t_scores.append(result['scores'][0])
                bleurt_scores[i].append(max(_t_scores))
            pbar.update(1)
    return bleurt_scores

def eval_bleu4(
    hypotheses: List[List[str]],
    references: List[List[str]],
    ) -> List[float]:
    """
    Evaluate the hypothesis and reference using the metric.

    Args:
        hypotheses: the hypotheses
        references: the references

    Returns:
        A list of bleu scores, in same shape with hypotheses.
    """
    print("Evaluating bleu4")
    assert len(hypotheses) == len(references)
    # tokenization
    nlp = spacy.load("en_core_web_sm")
    disable_pipes = list(nlp.pipe_names)
    disable_pipes.remove('tagger')
    nlp.disable_pipes(*disable_pipes)
    for i in tqdm(range(len(hypotheses)), desc="Tokenizing"):
        for j in range(len(hypotheses[i])):
            hypotheses[i][j] = " ".join([token.text for token in nlp(hypotheses[i][j])])
        for j in range(len(references[i])):
            references[i][j] = " ".join([token.text for token in nlp(references[i][j])])

    bleu4_scorer = Bleu(4)
    gts = {}
    res = {}
    hypo_ids_per_ref = []
    id = 0

    for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
        hypo_ids_per_ref.append([])
        for hypo in hypo_group:
            gts[id] = ref
            res[id] = [hypo]
            hypo_ids_per_ref[i].append(id)
            id += 1

    score, scores = bleu4_scorer.compute_score(gts, res)
    for method in zip(("Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"), score):
        print("%s: %0.3f" % method)
    bleu4_scores = scores[3]
    bleu4_scores = [[bleu4_scores[hypo_id]*100 for hypo_id in hypo_ids] for hypo_ids in hypo_ids_per_ref]
    return bleu4_scores

def eval_chrf(
    hypotheses: List[List[str]],
    references: List[List[str]],
    ) -> List[float]:
    """
    Evaluate the hypothesis and reference using the metric.

    Args:
        hypotheses: the hypotheses
        references: the references
    """
    print("Evaluating chrf")
    assert len(hypotheses) == len(references)
    from sacrebleu import sentence_chrf
    chrf_scores = []
    with tqdm(total=len(hypotheses), desc="Evaluating chrf") as pbar:
        for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
            chrf_scores.append([])
            for hypo in hypo_group:
                chrf_scores[i].append(sentence_chrf(hypo, ref).score)
            pbar.update(1)
    return chrf_scores

def eval_cider(
    hypotheses: List[List[str]],
    references: List[List[str]],
    ) -> List[float]:
    """
    Evaluate the hypothesis and reference using the metric.

    Args:
        hypotheses: the hypotheses
        references: the references
    """
    print("Evaluating cider")
    assert len(hypotheses) == len(references)

    # tokenization
    nlp = spacy.load("en_core_web_sm")
    disable_pipes = list(nlp.pipe_names)
    disable_pipes.remove('tagger')
    nlp.disable_pipes(*disable_pipes)
    for i in tqdm(range(len(hypotheses)), desc="Tokenizing"):
        for j in range(len(hypotheses[i])):
            hypotheses[i][j] = " ".join([token.text for token in nlp(hypotheses[i][j])])
        for j in range(len(references[i])):
            references[i][j] = " ".join([token.text for token in nlp(references[i][j])])

    cider_scorer = Cider()
    gts = {}
    res = {}
    hypo_ids_per_ref = []
    id = 0

    for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
        hypo_ids_per_ref.append([])
        for hypo in hypo_group:
            gts[id] = ref
            res[id] = [hypo]
            hypo_ids_per_ref[i].append(id)
            id += 1

    score, scores = cider_scorer.compute_score(gts, res)
    cider_scores = [[scores[hypo_id]*10 for hypo_id in hypo_ids] for hypo_ids in hypo_ids_per_ref]
    return cider_scores

def eval_spice(
    hypotheses: List[List[str]],
    references: List[List[str]]
    ) -> List[float]:
    """
    Evaluate the hypothesis and reference using the metric.

    Args:
        hypotheses: the hypotheses
        references: the references
    """
    print("Evaluating spice")
    assert len(hypotheses) == len(references)
    # tokenization
    nlp = spacy.load("en_core_web_sm")
    disable_pipes = list(nlp.pipe_names)
    disable_pipes.remove('tagger')
    nlp.disable_pipes(*disable_pipes)
    for i in tqdm(range(len(hypotheses)), desc="Tokenizing"):
        for j in range(len(hypotheses[i])):
            hypotheses[i][j] = " ".join([token.text for token in nlp(hypotheses[i][j])])
        for j in range(len(references[i])):
            references[i][j] = " ".join([token.text for token in nlp(references[i][j])])

    spice_scorer = Spice()
    gts = {}
    res = {}
    hypo_ids_per_ref = []
    id = 0
    for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
        hypo_ids_per_ref.append([])
        for hypo in hypo_group:
            gts[id] = ref
            res[id] = [hypo]
            hypo_ids_per_ref[i].append(id)
            id += 1

    score, scores = spice_scorer.compute_score(gts, res)
    spice_scores = [[scores[hypo_id]['All']['f']*100.0 for hypo_id in hypo_ids] for hypo_ids in hypo_ids_per_ref]
    return spice_scores    

def eval_prism(
    hypotheses: List[List[str]],
    references: List[List[str]],
) -> List[float]:
    # We do not src
    from prism import Prism
    from pathlib import Path
    model_path = str(Path(os.path.abspath(__file__)).parent / Path("models/m39v1"))
    prism = Prism(model_dir= model_path ,lang='en',temperature=1.0)
    prism_scores = []
    with tqdm(total=len(hypotheses), desc="Evaluating prism") as pbar:
        for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
            # prism_scores.append([])
            # for hypo in hypo_group:
            prism_scores.append(list(prism.score(hypo_group, ref,segment_scores=True)[-1]))
            pbar.update(1)
    return prism_scores

def eval_comet_da(
    hypotheses: List[List[str]],
    references: List[List[str]],
    sources: List[str],
) -> List[float]:
    from comet import download_model, load_from_checkpoint
    model_path = download_model('Unbabel/wmt22-comet-da')
    model = load_from_checkpoint(model_path)
    samples = []
    assert len(hypotheses) == len(references) == len(sources)
    for i, (hypo_group, ref_group, source) in enumerate(zip(hypotheses, references, sources)):
        for hypo in hypo_group:
            samples.append({
                "src": source,
                "ref": ref_group,
                "mt": hypo,
            })
    batch_size = 8
    res = model.predict(samples, gpus=1, batch_size=batch_size)
    comets = res[0]
    wrapped_comets = []
    idx = 0
    for i in range(len(hypotheses)):
        wrapped_comets.append([])
        for j in range(len(hypotheses[i])):
            wrapped_comets[i].append(comets[idx])
            idx += 1
    return wrapped_comets

def eval_cometkiwi_da(
    hypotheses: List[List[str]],
    sources: List[str],
) -> List[float]:
    from comet import download_model, load_from_checkpoint
    model_path = download_model('Unbabel/wmt22-cometkiwi-da')
    model = load_from_checkpoint(model_path)
    samples = []
    assert len(hypotheses) == len(sources)
    for i, (hypo_group, source) in enumerate(zip(hypotheses, sources)):
        for hypo in zip(hypo_group):
            samples.append({
                "src": source,
                "mt": hypo,
            })
    batch_size = 8
    res = model.predict(samples, gpus=1, batch_size=batch_size)
    comets = res[0]
    wrapped_comets = []
    idx = 0
    for i in range(len(hypotheses)):
        wrapped_comets.append([])
        for j in range(len(hypotheses[i])):
            wrapped_comets[i].append(comets[idx])
            idx += 1
    return wrapped_comets


def eval_bertscore(
    hypotheses: List[List[str]],
    references: List[List[str]],
    model_type="bert-base-multilingual-cased",
    lang="en",
    ) -> List[float]:
    """
    Evaluate the hypothesis and reference using bertscore.
    BertScore officially recommends using microsoft/deberta-xlarge-mnli as the model.
    the default multilingual model is bert-base-multilingual-cased.

    Args:
        hypotheses: the hypotheses
        references: the references
    """
    import bert_score
    print("Evaluating bertscore")
    assert len(hypotheses) == len(references)
    flatten_hypotheses = []
    flatten_references = []
    for i in range(len(hypotheses)):
        flatten_hypotheses.extend(hypotheses[i])
        flatten_references.extend([references[i]] * len(hypotheses[i]))
    
    P, R, F1 = bert_score.score(flatten_hypotheses, flatten_references, lang=lang, verbose=True, model_type=model_type, batch_size=16)
    flatten_scores = F1.numpy().tolist()
    scores = []
    idx = 0
    for i in range(len(hypotheses)):
        scores.append([])
        for j in range(len(hypotheses[i])):
            scores[i].append(flatten_scores[idx])
            idx += 1
    return scores

def eval_bartscore(
    hypotheses: List[List[str]],
    references: List[List[str]],
    metric_name = "bart_score",
    ) -> List[float]:
    """
    Evaluate the hypothesis and reference using the metric.

    Args:
        hypotheses: the hypotheses
        references: the references
        metric_name: bart_score or bart_score_cnn or bart_score_para
    """
    assert len(hypotheses) == len(references)
    from bart_score import BARTScorer
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    if metric_name == "bart_score_cnn":
        bart_scorer = BARTScorer(device=device, checkpoint='facebook/bart-large-cnn')
    elif metric_name == "bart_score_para":
        bart_scorer = BARTScorer(device=device, checkpoint='facebook/bart-large-cnn')
        from pathlib import Path
        model_path = Path(os.path.abspath(__file__)).parent / Path("models/bart_score_para.pth")
        if not model_path.exists():
            model_path.parent.mkdir(parents=True, exist_ok=True)
            print("Downloading bart_score_para model")
            os.system("gdown https://drive.google.com/uc?id=1_7JfF7KOInb7ZrxKHIigTMR4ChVET01m -O %s" % model_path)
        else:
            print("bart_score_para model exists")
        bart_scorer.load(model_path)
    elif metric_name == "bart_score":
        bart_scorer = BARTScorer(device=device, checkpoint='facebook/bart-large')
    else:
        raise ValueError("metric_name should be bart_score or bart_score_cnn or bart_score_para")

    bart_scores = []
    with tqdm(total=len(hypotheses), desc="Evaluating bartscore") as pbar:
        for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
            _temp_scores = []
            if not isinstance(ref, list):
                ref = [ref]
            for ref_i in ref:
                hypo_ref_scores = np.array(bart_scorer.score(hypo_group, [ref_i]*len(hypo_group), batch_size=4))
                ref_hypo_scores = np.array(bart_scorer.score([ref_i]*len(hypo_group), hypo_group, batch_size=4))
                _temp_scores.append(((hypo_ref_scores + ref_hypo_scores) / 2).tolist())
                # _temp_scores.append(hypo_ref_scores.tolist())
            # get max ref score
            _temp_scores = np.array(_temp_scores).max(axis=0)
            bart_scores.append(_temp_scores)
            pbar.update(1)
            assert len(bart_scores[i]) == len(hypo_group)
    return bart_scores

def eval_bartscore_src_hypo(
    hypotheses: List[List[str]],
    sources: List[str],
    metric_name = "bart_score",
    ) -> List[float]:
    """
    Evaluate the hypothesis with source using the metric.

    Args:
        hypotheses: the hypotheses
        sources: the sources
        metric_name: bart_score or bart_score_cnn or bart_score_para
    """
    assert len(hypotheses) == len(sources)
    from bart_score import BARTScorer
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    if metric_name == "bart_score_cnn":
        bart_scorer = BARTScorer(device=device, checkpoint='facebook/bart-large-cnn')
    elif metric_name == "bart_score_para":
        bart_scorer = BARTScorer(device=device, checkpoint='facebook/bart-large-cnn')
        from pathlib import Path
        model_path = Path(os.path.abspath(__file__)).parent / Path("models/bart_score_para.pth")
        if not model_path.exists():
            model_path.parent.mkdir(parents=True, exist_ok=True)
            print("Downloading bart_score_para model")
            os.system("gdown https://drive.google.com/uc?id=1_7JfF7KOInb7ZrxKHIigTMR4ChVET01m -O %s" % model_path)
        else:
            print("bart_score_para model exists")
        bart_scorer.load(model_path)
    else:
        bart_scorer = BARTScorer(device=device, checkpoint='facebook/bart-large')

    bart_scores = []
    with tqdm(total=len(hypotheses), desc="Evaluating bartscore") as pbar:
        for i, (hypo_group, source) in enumerate(zip(hypotheses, sources)):
            src_hypo_scores = bart_scorer.score([source]*len(hypo_group), hypo_group, batch_size=4)
            bart_scores.append(src_hypo_scores)
            pbar.update(1)
            assert len(bart_scores[i]) == len(hypo_group)
    return bart_scores

def eval_unieval(
    hypotheses: List[List[str]],
    references: List[List[str]],
    sources: List[str],
    task: str,    
):
    from unieval.utils import convert_to_json
    from unieval.metric.evaluator import get_evaluator
    assert task in ['summarization', 'dialogue', 'data2text', 'fact']
    if isinstance(hypotheses[0], list):
        # flatten
        flatten_hypotheses = []
        flatten_references = [] if references is not None else None
        flatten_sources = [] if sources is not None else None
        for i in range(len(hypotheses)):
            for hypo in hypotheses[i]:
                if hypo:
                    flatten_hypotheses.append(hypo)
                    if references is not None:
                        if isinstance(references[i], list):
                            flatten_references.append(references[i][0])
                        elif isinstance(references[i], str):
                            flatten_references.append(references[i])
                        else:
                            raise ValueError("references should be a list of list of str or a list of str")
                    if sources is not None:
                        flatten_sources.append(sources[i])
        data = convert_to_json(output_list=flatten_hypotheses, src_list=flatten_sources, ref_list=flatten_references)
    elif isinstance(hypotheses[0], str):
        output_list = [x for x in hypotheses if x]
        src_list = [x for i, x in enumerate(sources) if hypotheses[i]]
        ref_list = [x for i, x in enumerate(references) if hypotheses[i]]
        data = convert_to_json(output_list=output_list, src_list=src_list, ref_list=ref_list)
    else:
        raise ValueError("hypotheses should be a list of list of str or a list of str")
    # compute
    evaluator = get_evaluator(task)
    eval_scores = evaluator.evaluate(data, print_result=True)
    aspect_scores = defaultdict(list)
    # unflatten
    aspects = list(eval_scores[0].keys())
    if isinstance(hypotheses[0], list):
        idx = 0
        for i, hypo_group in enumerate(hypotheses):
            for aspect in aspects:
                aspect_scores[aspect].append([])
            for hypo in hypo_group:
                if hypo:
                    for aspect in aspects:
                        aspect_scores[aspect][i].append(eval_scores[idx][aspect])
                    idx += 1
                else:
                    for aspect in aspects:
                        aspect_scores[aspect][i].append(0)
    else:
        for aspect in aspects:
            idx = 0
            for hypo in hypotheses:
                if hypo:
                    aspect_scores[aspect].append(eval_scores[idx][aspect])
                    idx += 1
                else:
                    aspect_scores[aspect].append(0)
    return aspect_scores

def eval_instructscore(
    hypotheses: List[List[str]],
    references: List[List[str]],
):
    from InstructScore import InstructScore
    scorer = InstructScore()
    assert len(hypotheses) == len(references)
    flatten_hypotheses = []
    flatten_references = []
    for i in range(len(hypotheses)):
        flatten_hypotheses.extend(hypotheses[i])
        if isinstance(references[i], list):
            flatten_references.extend([references[i][0]] * len(hypotheses[i]))
        elif isinstance(references[i], str):
            flatten_references.extend([references[i]] * len(hypotheses[i]))
        else:
            raise ValueError("references should be a list of list of str or a list of str")
    batch_outputs, scores_ls = scorer.score(flatten_references, flatten_hypotheses)
    idx = 0
    instruct_scores = []
    for i in range(len(hypotheses)):
        instruct_scores.append([])
        for j in range(len(hypotheses[i])):
            instruct_scores[i].append(scores_ls[idx])
            idx += 1
    return instruct_scores

def eval_gptscore_ref_ist(
    hypotheses: List[List[str]],
    references: List[List[str]],
    task="summarization",
    metric_name="flan_base_score",
):
    """
    Instruction (IST) only version gptscore
    """
    asp_definitions = {
        "Semantic Coverage (COV)": "How many semantic content units from the reference text are covered by the generated text?",
        "Factuality (FAC)": "Does the generated text preserve the factual statements of the source text?",
        "Consistency (CON)": "Is the generated text consistent in the information it provides?",
        "Informativeness (INF)": "How well does the generated text capture the key ideas of its source text?",
        "Coherence (COH)": "How much does the generated text make sense?",
        "Relevance (REL)": "How well is the generated text relevant to its source text?",
        "Fluency (FLU)": "Is the generated text well-written and grammatical?",
        "Accuracy (ACC)": "Are there inaccuracies, missing, or unfactual content in the generated text?",
        "Multidimensional Quality Metrics (MQM)": "How is the overall quality of the generated text?",
        "Interest (INT)": "Is the generated text interesting?",
        "Engagement (ENG)": "Is the generated text engaging?",
        "Specific (SPE)": "Is the generated text generic or specific to the source text?",
        "Correctness (COR)": "Is the generated text correct or was there a misunderstanding of the source text?",
        "Semantically appropriate (SEM)": "Is the generated text semantically appropriate?",
        "Understandability (UND)": "Is the generated text understandable?",
        "Error Recovery (ERR)": "Is the system able to recover from errors that it makes?",
        "Diversity (DIV)": "Is there diversity in the system responses?",
        "Depth (DEP)": "Does the system discuss topics in depth?",
        "Likeability (LIK)": "Does the system display a likeable personality?",
        "Flexibility (FLE)": "Is the system flexible and adaptable to the user and their interests?",
        "Inquisitiveness (INQ)": "Is the system inquisitive throughout the conversation?",
    }
    asp_names = list(asp_definitions.keys())
    task_asp_map = {
        "Summ": asp_names[:7],
        "D2T": [asp_names[3]] + asp_names[5:7],
        "Dial": asp_names[2:4] + asp_names[5:7] + asp_names[9:],
        "MT": asp_names[6:9],
    }
    assert task in task_asp_map.keys()
    final_gpt_scores = {}
    from mosestokenizer import MosesDetokenizer
    detokenizer = MosesDetokenizer('en')
    def add_dot(text):
        if len(text.strip()) == 0:
            return '.'
        if text.strip()[-1] != '.':
            text = text.strip() + ' .'
        new_text = text
        return new_text
    def detokenize(text: str):
        # words = text.split(" ")
        words = text.split()
        return detokenizer(words)
    if "flan" in metric_name:
        from flan_score import FLANScorer
        import time
        metric2checkpoint = {
            "flan_small_score": "google/flan-t5-small",
            "flan_base_score": "google/flan-t5-base",
            "flan_large_score": "google/flan-t5-large",
            "flan_xl_score": "google/flan-t5-xl",
            "flan_xxl_score": "google/flan-t5-xxl",
        }
        print('metric_name: ', metric_name)
        checkpoint = metric2checkpoint[metric_name]
        device = "cuda" if torch.cuda.is_available() else "cpu"
        flan_scorer = FLANScorer(device=device, checkpoint=checkpoint)
        print(f'FLANScorer setup finished. Begin calculating FLANScorer.')
        start = time.time()
        for e_asp in task_asp_map[task]:
            print("evaluting ", e_asp)
            # Evaluation is cheap when using non-GPT3 models, so here, we evaluate all aspects by default.
            print('num of examples: ', len(hypotheses))
            asp_df = asp_definitions[e_asp]
            asp_df = asp_df.strip().replace(':', '. ')
            print('asp_df: ', asp_df)
            # perfix
            prefix = asp_df + '\n'
            template = "XXXXX In other words , YYYYY"
            ## ref->hypo
            gpt_scores = []

            with tqdm(total=len(hypotheses), desc="Evaluating gptscore (Flan)") as pbar:
                for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
                    _temp_scores = []
                    if not isinstance(ref, list):
                        ref = [ref]
                    for ref_i in ref:
                        hyps = [add_dot(detokenize(hypo)) for hypo in hypo_group]
                        ref_i_ = add_dot(detokenize(ref_i))
                        hypo_ref_scores = np.array(flan_scorer.score(
                            [prefix+template.replace('XXXXX', x).replace('YYYYY', '') for x in hyps], 
                            [ref_i_]*len(hypo_group), batch_size=4)
                        )
                        ref_hypo_scores = np.array(flan_scorer.score(
                            [prefix+template.replace('XXXXX', ref_i_).replace('YYYYY', '')]*len(hypo_group), hyps, batch_size=4))
                        _temp_scores.append(((hypo_ref_scores + ref_hypo_scores) / 2).tolist())
                        # _temp_scores.append(hypo_ref_scores.tolist())
                    # get max ref score
                    _temp_scores = np.array(_temp_scores).max(axis=0)
                    gpt_scores.append(_temp_scores)
                    pbar.update(1)
                    assert len(gpt_scores[i]) == len(hypo_group)
            final_gpt_scores[e_asp] = gpt_scores
        end = time.time()
        print(f'FLANScorer finished. Time: {end-start}')
    else:
        raise NotImplementedError
    
    # final_gpt_scores["avg_all_asps"] = np.mean([np.array(final_gpt_scores[asp]) for asp in task_asp_map[task]], axis=0).tolist()
    scores = [final_gpt_scores[asp] for asp in task_asp_map[task]]
    avg_scores = [sum(x) / len(x) for x in zip(*scores)]
    final_gpt_scores["avg_all_asps"] = avg_scores
    return final_gpt_scores

def eval_gptscore_src_ist(
    hypotheses: List[List[str]],
    sources: List[str],
    task="summarization",
    metric_name="flan_base_score",
):
    """
    Instruction (IST) only version gptscore
    """
    asp_definitions = {
        "Semantic Coverage (COV)": "How many semantic content units from the reference text are covered by the generated text?",
        "Factuality (FAC)": "Does the generated text preserve the factual statements of the source text?",
        "Consistency (CON)": "Is the generated text consistent in the information it provides?",
        "Informativeness (INF)": "How well does the generated text capture the key ideas of its source text?",
        "Coherence (COH)": "How much does the generated text make sense?",
        "Relevance (REL)": "How well is the generated text relevant to its source text?",
        "Fluency (FLU)": "Is the generated text well-written and grammatical?",
        "Accuracy (ACC)": "Are there inaccuracies, missing, or unfactual content in the generated text?",
        "Multidimensional Quality Metrics (MQM)": "How is the overall quality of the generated text?",
        "Interest (INT)": "Is the generated text interesting?",
        "Engagement (ENG)": "Is the generated text engaging?",
        "Specific (SPE)": "Is the generated text generic or specific to the source text?",
        "Correctness (COR)": "Is the generated text correct or was there a misunderstanding of the source text?",
        "Semantically appropriate (SEM)": "Is the generated text semantically appropriate?",
        "Understandability (UND)": "Is the generated text understandable?",
        "Error Recovery (ERR)": "Is the system able to recover from errors that it makes?",
        "Diversity (DIV)": "Is there diversity in the system responses?",
        "Depth (DEP)": "Does the system discuss topics in depth?",
        "Likeability (LIK)": "Does the system display a likeable personality?",
        "Flexibility (FLE)": "Is the system flexible and adaptable to the user and their interests?",
        "Inquisitiveness (INQ)": "Is the system inquisitive throughout the conversation?",
    }
    asp_names = list(asp_definitions.keys())
    task_asp_map = {
        "Summ": asp_names[:7],
        "D2T": [asp_names[3]] + asp_names[5:7],
        "Dial": asp_names[2:4] + asp_names[5:7] + asp_names[9:],
        "MT": asp_names[6:9],
    }
    assert task in task_asp_map.keys()
    final_gpt_scores = {}
    from mosestokenizer import MosesDetokenizer
    detokenizer = MosesDetokenizer('en')
    def add_dot(text):
        if len(text.strip()) == 0:
            return '.'
        if text.strip()[-1] != '.':
            text = text.strip() + ' .'
        new_text = text
        return new_text
    def detokenize(text: str):
        # words = text.split(" ")
        words = text.split()
        return detokenizer(words)
    if "flan" in metric_name:
        from flan_score import FLANScorer
        import time
        metric2checkpoint = {
            "flan_small_score": "google/flan-t5-small",
            "flan_base_score": "google/flan-t5-base",
            "flan_large_score": "google/flan-t5-large",
            "flan_xl_score": "google/flan-t5-xl",
            "flan_xxl_score": "google/flan-t5-xxl",
        }
        print('metric_name: ', metric_name)
        checkpoint = metric2checkpoint[metric_name]
        device = "cuda" if torch.cuda.is_available() else "cpu"
        flan_scorer = FLANScorer(device=device, checkpoint=checkpoint)
        print(f'FLANScorer setup finished. Begin calculating FLANScorer.')
        start = time.time()
        for e_asp in task_asp_map[task]:
            print("evaluting ", e_asp)
            # Evaluation is cheap when using non-GPT3 models, so here, we evaluate all aspects by default.
            print('num of examples: ', len(hypotheses))
            asp_df = asp_definitions[e_asp]
            asp_df = asp_df.strip().replace(':', '. ')
            print('asp_df: ', asp_df)
            # perfix
            prefix = asp_df + '\n'
            template = "XXXXX In other words , YYYYY"
            ## ref->hypo
            gpt_scores = []

            with tqdm(total=len(hypotheses), desc="Evaluating gptscore (Flan)") as pbar:
                for i, (hypo_group, source) in enumerate(zip(hypotheses, sources)):
                    _temp_scores = []
                    hyps = [add_dot(detokenize(hypo)) for hypo in hypo_group]
                    src = add_dot(detokenize(source))
                    src_hypo_scores = np.array(flan_scorer.score(
                        [prefix+template.replace('XXXXX', src).replace('YYYYY', '')]*len(hypo_group), hyps, batch_size=4))
                    gpt_scores.append(src_hypo_scores)
                    pbar.update(1)
                    assert len(gpt_scores[i]) == len(hypo_group)
            final_gpt_scores[e_asp] = gpt_scores
        end = time.time()
        print(f'FLANScorer finished. Time: {end-start}')
    else:
        raise NotImplementedError
    # final_gpt_scores["avg_all_asps"] = np.mean([np.array(final_gpt_scores[asp]) for asp in task_asp_map[task]], axis=0).tolist()
    scores = [final_gpt_scores[asp] for asp in task_asp_map[task]]
    avg_scores = [sum(x) / len(x) for x in zip(*scores)]
    final_gpt_scores["avg_all_asps"] = avg_scores
    return final_gpt_scores

def eval_gptscore_ref_ist(
    hypotheses: List[List[str]],
    references: List[List[str]],
    task="summarization",
    metric_name="flan_base_score",
):
    """
    Instruction (IST) only version gptscore
    """
    asp_definitions = {
        "Semantic Coverage (COV)": "How many semantic content units from the reference text are covered by the generated text?",
        "Factuality (FAC)": "Does the generated text preserve the factual statements of the source text?",
        "Consistency (CON)": "Is the generated text consistent in the information it provides?",
        "Informativeness (INF)": "How well does the generated text capture the key ideas of its source text?",
        "Coherence (COH)": "How much does the generated text make sense?",
        "Relevance (REL)": "How well is the generated text relevant to its source text?",
        "Fluency (FLU)": "Is the generated text well-written and grammatical?",
        "Accuracy (ACC)": "Are there inaccuracies, missing, or unfactual content in the generated text?",
        "Multidimensional Quality Metrics (MQM)": "How is the overall quality of the generated text?",
        "Interest (INT)": "Is the generated text interesting?",
        "Engagement (ENG)": "Is the generated text engaging?",
        "Specific (SPE)": "Is the generated text generic or specific to the source text?",
        "Correctness (COR)": "Is the generated text correct or was there a misunderstanding of the source text?",
        "Semantically appropriate (SEM)": "Is the generated text semantically appropriate?",
        "Understandability (UND)": "Is the generated text understandable?",
        "Error Recovery (ERR)": "Is the system able to recover from errors that it makes?",
        "Diversity (DIV)": "Is there diversity in the system responses?",
        "Depth (DEP)": "Does the system discuss topics in depth?",
        "Likeability (LIK)": "Does the system display a likeable personality?",
        "Flexibility (FLE)": "Is the system flexible and adaptable to the user and their interests?",
        "Inquisitiveness (INQ)": "Is the system inquisitive throughout the conversation?",
    }
    asp_names = list(asp_definitions.keys())
    task_asp_map = {
        "Summ": asp_names[:7],
        "D2T": [asp_names[3]] + asp_names[5:7],
        "Dial": asp_names[2:4] + asp_names[5:7] + asp_names[9:],
        "MT": asp_names[6:9],
    }
    assert task in task_asp_map.keys()
    final_gpt_scores = {}
    from mosestokenizer import MosesDetokenizer
    detokenizer = MosesDetokenizer('en')
    def add_dot(text):
        if len(text.strip()) == 0:
            return '.'
        if text.strip()[-1] != '.':
            text = text.strip() + ' .'
        new_text = text
        return new_text
    def detokenize(text: str):
        # words = text.split(" ")
        words = text.split()
        return detokenizer(words)
    if "flan" in metric_name:
        from flan_score import FLANScorer
        import time
        metric2checkpoint = {
            "flan_small_score": "google/flan-t5-small",
            "flan_base_score": "google/flan-t5-base",
            "flan_large_score": "google/flan-t5-large",
            "flan_xl_score": "google/flan-t5-xl",
            "flan_xxl_score": "google/flan-t5-xxl",
        }
        print('metric_name: ', metric_name)
        checkpoint = metric2checkpoint[metric_name]
        device = "cuda" if torch.cuda.is_available() else "cpu"
        flan_scorer = FLANScorer(device=device, checkpoint=checkpoint)
        print(f'FLANScorer setup finished. Begin calculating FLANScorer.')
        start = time.time()
        for e_asp in task_asp_map[task]:
            print("evaluting ", e_asp)
            # Evaluation is cheap when using non-GPT3 models, so here, we evaluate all aspects by default.
            print('num of examples: ', len(hypotheses))
            asp_df = asp_definitions[e_asp]
            asp_df = asp_df.strip().replace(':', '. ')
            print('asp_df: ', asp_df)
            # perfix
            prefix = asp_df + '\n'
            template = "XXXXX In other words , YYYYY"
            ## ref->hypo
            gpt_scores = []

            with tqdm(total=len(hypotheses), desc="Evaluating gptscore (Flan)") as pbar:
                for i, (hypo_group, ref) in enumerate(zip(hypotheses, references)):
                    _temp_scores = []
                    if not isinstance(ref, list):
                        ref = [ref]
                    for ref_i in ref:
                        hyps = [add_dot(detokenize(hypo)) for hypo in hypo_group]
                        ref_i_ = add_dot(detokenize(ref_i))
                        hypo_ref_scores = np.array(flan_scorer.score(
                            [prefix+template.replace('XXXXX', x).replace('YYYYY', '') for x in hyps], 
                            [ref_i_]*len(hypo_group), batch_size=4)
                        )
                        ref_hypo_scores = np.array(flan_scorer.score(
                            [prefix+template.replace('XXXXX', ref_i_).replace('YYYYY', '')]*len(hypo_group), hyps, batch_size=4))
                        _temp_scores.append(((hypo_ref_scores + ref_hypo_scores) / 2).tolist())
                        # _temp_scores.append(hypo_ref_scores.tolist())
                    # get max ref score
                    _temp_scores = np.array(_temp_scores).max(axis=0)
                    gpt_scores.append(_temp_scores)
                    pbar.update(1)
                    assert len(gpt_scores[i]) == len(hypo_group)
            final_gpt_scores[e_asp] = gpt_scores
        end = time.time()
        print(f'FLANScorer finished. Time: {end-start}')
    else:
        raise NotImplementedError
    # final_gpt_scores["avg_all_asps"] = np.mean([np.array(final_gpt_scores[asp]) for asp in task_asp_map[task]], axis=0).tolist()
    scores = [final_gpt_scores[asp] for asp in task_asp_map[task]]
    avg_scores = [sum(x) / len(x) for x in zip(*scores)]
    final_gpt_scores["avg_all_asps"] = avg_scores
    return final_gpt_scores

def eval_gptscore_src_ist(
    hypotheses: List[List[str]],
    sources: List[str],
    task="summarization",
    metric_name="flan_base_score",
):
    """
    Instruction (IST) only version gptscore
    """
    asp_definitions = {
        "Semantic Coverage (COV)": "How many semantic content units from the reference text are covered by the generated text?",
        "Factuality (FAC)": "Does the generated text preserve the factual statements of the source text?",
        "Consistency (CON)": "Is the generated text consistent in the information it provides?",
        "Informativeness (INF)": "How well does the generated text capture the key ideas of its source text?",
        "Coherence (COH)": "How much does the generated text make sense?",
        "Relevance (REL)": "How well is the generated text relevant to its source text?",
        "Fluency (FLU)": "Is the generated text well-written and grammatical?",
        "Accuracy (ACC)": "Are there inaccuracies, missing, or unfactual content in the generated text?",
        "Multidimensional Quality Metrics (MQM)": "How is the overall quality of the generated text?",
        "Interest (INT)": "Is the generated text interesting?",
        "Engagement (ENG)": "Is the generated text engaging?",
        "Specific (SPE)": "Is the generated text generic or specific to the source text?",
        "Correctness (COR)": "Is the generated text correct or was there a misunderstanding of the source text?",
        "Semantically appropriate (SEM)": "Is the generated text semantically appropriate?",
        "Understandability (UND)": "Is the generated text understandable?",
        "Error Recovery (ERR)": "Is the system able to recover from errors that it makes?",
        "Diversity (DIV)": "Is there diversity in the system responses?",
        "Depth (DEP)": "Does the system discuss topics in depth?",
        "Likeability (LIK)": "Does the system display a likeable personality?",
        "Flexibility (FLE)": "Is the system flexible and adaptable to the user and their interests?",
        "Inquisitiveness (INQ)": "Is the system inquisitive throughout the conversation?",
    }
    asp_names = list(asp_definitions.keys())
    task_asp_map = {
        "Summ": asp_names[:7],
        "D2T": [asp_names[3]] + asp_names[5:7],
        "Dial": asp_names[2:4] + asp_names[5:7] + asp_names[9:],
        "MT": asp_names[6:9],
    }
    assert task in task_asp_map.keys()
    final_gpt_scores = {}
    from mosestokenizer import MosesDetokenizer
    detokenizer = MosesDetokenizer('en')
    def add_dot(text):
        if len(text.strip()) == 0:
            return '.'
        if text.strip()[-1] != '.':
            text = text.strip() + ' .'
        new_text = text
        return new_text
    def detokenize(text: str):
        # words = text.split(" ")
        words = text.split()
        return detokenizer(words)
    if "flan" in metric_name:
        from flan_score import FLANScorer
        import time
        metric2checkpoint = {
            "flan_small_score": "google/flan-t5-small",
            "flan_base_score": "google/flan-t5-base",
            "flan_large_score": "google/flan-t5-large",
            "flan_xl_score": "google/flan-t5-xl",
            "flan_xxl_score": "google/flan-t5-xxl",
        }
        print('metric_name: ', metric_name)
        checkpoint = metric2checkpoint[metric_name]
        device = "cuda" if torch.cuda.is_available() else "cpu"
        flan_scorer = FLANScorer(device=device, checkpoint=checkpoint)
        print(f'FLANScorer setup finished. Begin calculating FLANScorer.')
        start = time.time()
        for e_asp in task_asp_map[task]:
            print("evaluting ", e_asp)
            # Evaluation is cheap when using non-GPT3 models, so here, we evaluate all aspects by default.
            print('num of examples: ', len(hypotheses))
            asp_df = asp_definitions[e_asp]
            asp_df = asp_df.strip().replace(':', '. ')
            print('asp_df: ', asp_df)
            # perfix
            prefix = asp_df + '\n'
            template = "XXXXX In other words , YYYYY"
            ## ref->hypo
            gpt_scores = []

            with tqdm(total=len(hypotheses), desc="Evaluating gptscore (Flan)") as pbar:
                for i, (hypo_group, source) in enumerate(zip(hypotheses, sources)):
                    _temp_scores = []
                    hyps = [add_dot(detokenize(hypo)) for hypo in hypo_group]
                    src = add_dot(detokenize(source))
                    src_hypo_scores = np.array(flan_scorer.score(
                        [prefix+template.replace('XXXXX', src).replace('YYYYY', '')]*len(hypo_group), hyps, batch_size=4))
                    gpt_scores.append(src_hypo_scores)
                    pbar.update(1)
                    assert len(gpt_scores[i]) == len(hypo_group)
            final_gpt_scores[e_asp] = gpt_scores
        end = time.time()
        print(f'FLANScorer finished. Time: {end-start}')
    else:
        raise NotImplementedError
    # final_gpt_scores["avg_all_asps"] = np.mean([np.array(final_gpt_scores[asp]) for asp in task_asp_map[task]], axis=0).tolist()
    scores = [final_gpt_scores[asp] for asp in task_asp_map[task]]
    avg_scores = [sum(x) / len(x) for x in zip(*scores)]
    final_gpt_scores["avg_all_asps"] = avg_scores
    return final_gpt_scores
            
        

def compute_new_n_gram(source:str, candidate:str):
    """
        computer the new n-grams in the candidate compared to source text
    """
    # text
    text = source.lower()
    text_words = word_tokenize(text)
    text_bigrams = [[text_words[j], text_words[j + 1]] for j in range(len(text_words) - 1)]
    text_trigrams = [[text_words[j], text_words[j + 1], text_words[j + 2]] for j in range(len(text_words) - 2)]
    text_quadrigrams = [[text_words[j], text_words[j + 1], text_words[j + 2], text_words[j + 3]] for j in range(len(text_words) - 3)]

    # candidate
    candidate = candidate.lower().replace("<n>", " ")
    candidate_words = word_tokenize(candidate)

    unigrams, bigrams, trigrams, quadrigrams = 0, 0, 0, 0
    for j in range(len(candidate_words)):
        if not(candidate_words[j] in text_words):
            unigrams += 1
        if j < len(candidate_words) - 1:
            bigram = [candidate_words[j], candidate_words[j + 1]]
            if not(bigram in text_bigrams):
                bigrams += 1
        if j < len(candidate_words) - 2:
            trigram = [candidate_words[j], candidate_words[j + 1], candidate_words[j + 2]]
            if not(trigram in text_trigrams):
                trigrams += 1
        if j < len(candidate_words) - 3:
            quadrigram = [candidate_words[j], candidate_words[j + 1], candidate_words[j + 2], candidate_words[j + 3]]
            if not(quadrigram in text_quadrigrams):
                quadrigrams += 1
    new_unigram, new_bigram, new_trigram, new_quadrigram = 0, 0, 0, 0
    if len(candidate_words) > 0:
        new_unigram = unigrams / (len(candidate_words) - 0)
    if len(candidate_words) > 1:
        new_bigram = bigrams / (len(candidate_words) - 1)
    if len(candidate_words) > 2:
        new_trigram = trigrams / (len(candidate_words) - 2)
    if len(candidate_words) > 3:
        new_quadrigram = quadrigrams / (len(candidate_words) - 3)
    return new_unigram, new_bigram, new_trigram, new_quadrigram




def eval_novel_n_gram(
    sources: List[str],
    hypotheses: Union[List[List[str]], List[str]],
    ) -> List[float]:
    """
        evaluate the novel n-gram in the hypotheses compared to the origianl soiurce
    """
    print("Evaluating novel n-gram")
    assert len(hypotheses) == len(sources)
    for i in range(len(hypotheses)):
        if isinstance(hypotheses[i], str):
            hypotheses[i] = [hypotheses[i]]

    new_unigrams, new_bigrams, new_trigrams, new_quadrigrams = [], [], [], []
    for i, (source, hypo_group) in tqdm(enumerate(zip(sources, hypotheses)), desc="evaluate novel n-grams"):
        new_unigrams.append([])
        new_bigrams.append([])
        new_trigrams.append([])
        new_quadrigrams.append([])
        for hypo in hypo_group:
            new_unigram, new_bigram, new_trigram, new_quadrigram = \
                compute_new_n_gram(source, hypo)
            new_unigrams[i].append(new_unigram)
            new_bigrams[i].append(new_bigram)
            new_trigrams[i].append(new_trigram)
            new_quadrigrams[i].append(new_quadrigram)

    new_unigrams = np.array(new_unigrams)
    m_uni = 100 * np.mean(new_unigrams)
    new_bigrams = np.array(new_bigrams)
    m_bi = 100 * np.mean(new_bigrams)
    new_trigrams = np.array(new_trigrams)
    m_tri = 100 * np.mean(new_trigrams)
    new_quadrigrams = np.array(new_quadrigrams)
    m_quadri = 100 * np.mean(new_quadrigrams)
    print("New unigrams: {:.2f}, bigrams: {:.2f}, trigrams: {:.2f}, quadrigrams: {:.2f}".format(m_uni, m_bi, m_tri, m_quadri))
    # nested remove list with single element
    if all([len(score) == 1 for score in new_unigrams]):
        new_unigrams = [score[0] for score in new_unigrams]
    if all([len(score) == 1 for score in new_bigrams]):
        new_bigrams = [score[0] for score in new_bigrams]
    if all([len(score) == 1 for score in new_trigrams]):
        new_trigrams = [score[0] for score in new_trigrams]
    if all([len(score) == 1 for score in new_quadrigram]):
        new_quadrigram = [score[0] for score in new_quadrigram]
    return new_unigrams, new_bigrams, new_trigrams, new_quadrigrams

def eval_distinct_n_grams(texts:Union[List[List[str]], List[str]]):
    print("evaluating distinct n-grams")
    for i in range(len(texts)):
        if isinstance(texts[i], str):
            texts[i] = [texts[i]]

    uni_unigrams, uni_bigrams, uni_trigrams, uni_quadrigrams = [], [], [], []
    for i, text_group in tqdm(enumerate(texts), desc='evaluting distinct n-grams'):
        unigrams = []
        bigrams = []
        trigrams = []
        quadrigrams = []
        for text in text_group:
            text = text.lower()
            text_words = word_tokenize(text)
            text_bigrams = [(text_words[j], text_words[j + 1]) for j in range(len(text_words) - 1)]
            text_trigrams = [(text_words[j], text_words[j + 1], text_words[j + 2]) for j in range(len(text_words) - 2)]
            text_quadrigrams = [(text_words[j], text_words[j + 1], text_words[j + 2], text_words[j + 3]) for j in range(len(text_words) - 3)]
            unigrams.extend(text_words)
            bigrams.extend(text_bigrams)
            trigrams.extend(text_trigrams)
            quadrigrams.extend(text_quadrigrams)
        unigrams = set(unigrams)
        bigrams = set(unigrams)
        trigrams = set(trigrams)
        quadrigrams = set(quadrigrams)
        uni_unigrams.append(len(unigrams))
        uni_bigrams.append(len(bigrams))
        uni_trigrams.append(len(trigrams))
        uni_quadrigrams.append(len(quadrigrams))
    print(f"Mean unique 1-grams: {np.mean(uni_unigrams)}")
    print(f"Mean unique 2-grams: {np.mean(uni_bigrams)}")
    print(f"Mean unique 3-grams: {np.mean(uni_trigrams)}")
    print(f"Mean unique 4-grams: {np.mean(uni_quadrigrams)}")
    return uni_unigrams, uni_bigrams, uni_trigrams, uni_quadrigrams

def eval_self_bleu(texts:List[List[str]]):
    print("evaluating self bleu")
    for i in range(len(texts)):
        assert isinstance(texts[i], list)

    self_bleus = []
    for i, text_group in tqdm(enumerate(texts), desc='evaluting distinct n-grams'):
        group_self_bleus = []
        for j in range(len(text_group)):
            hypo = text_group[j]
            refs = text_group[:j] + text_group[j+1:]
            group_self_bleus.append(sentence_bleu(hypothesis=hypo, references=refs).score)
        self_bleus.append(np.mean(group_self_bleus))
    print(f"self BLEUs mean: {np.mean(self_bleus)}")
    return self_bleus

def _overall_eval_multi_process(data):
    s = psutil.Process(os.getpid())
    cpu_id = s.cpu_num()
    print("Worker {} is evaluating".format(cpu_id))
    return _overall_eval(*data)

def _overall_eval(candidates, targets, metrics:List[str], sources=None):
    do_flatten = False
    # deepcopy in case it will make change to the passed in candidates and targets
    candidates = deepcopy(candidates)
    targets = deepcopy(targets)
    assert len(candidates) == len(targets), f"candidates and targets should have the same length, but got {len(candidates)} and {len(targets)}"
    # if there are no available targets, return None
    if all([target == '' for target in targets]) or \
        all([target == [] for target in targets]):
        return {
            metric: [
                [0 for _ in range(len(candidates[i]))]
                for i in range(len(candidates))]
            for metric in metrics
        }
    for i in range(len(candidates)):
        if isinstance(candidates[i], str):
            do_flatten = True
            candidates[i] = [candidates[i]]
        if isinstance(targets[i], str):
            targets[i] = [targets[i]]

    assert isinstance(sources, list) and isinstance(sources[0], str), "sources should be a list of str"

    if "rouge" in metrics:
        metrics.remove("rouge")
        metrics.extend(['rouge1', 'rouge2', 'rougeL', 'rougeLsum'])
        logging.warning("Rouge is a group of metrics, using rouge1, rouge2, rougeL, rougeLsum instead")
    scores = {}
    rouge_tyeps = [metric for metric in metrics if metric.startswith('rouge')]
    if rouge_tyeps:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        rouge_scores = eval_rouge(_candidates, _targets, rouge_types=rouge_tyeps)
        scores.update(rouge_scores)
    if 'bleu' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        bleu_scores = eval_bleu(_candidates, _targets)
        scores.update({'bleu': bleu_scores})
    if 'bleu4' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        bleu4_scores = eval_bleu4(_candidates, _targets)
        scores.update({'bleu4': bleu4_scores})
    if 'bleurt' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        bleurt_scores = eval_bleurt(_candidates, _targets)
        scores.update({'bleurt': bleurt_scores})
    if 'cider' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        cider_scores = eval_cider(_candidates, _targets)
        scores.update({'cider': cider_scores})
    if 'spice' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        spice_scores = eval_spice(_candidates, _targets)
        scores.update({'spice': spice_scores})
    if 'bertscore' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        bert_scores = eval_bertscore(_candidates, _targets)
        scores.update({'bertscore': bert_scores})
    if 'chrf' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        chrf_scores = eval_chrf(_candidates, _targets)
        scores.update({'chrf': chrf_scores})
    if 'prism' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        prism_scores = eval_prism(_candidates, _targets)
        scores.update({'prism': prism_scores})
    if 'comet_da' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        comet_scores = eval_comet_da(_candidates, _targets, sources)
        scores.update({'comet_da': comet_scores})
    if 'cometkiwi_da' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        comet_scores = eval_cometkiwi_da(_candidates, sources)
        scores.update({'cometkiwi_da': comet_scores})
    
    if 'bart_score' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        bart_scores = eval_bartscore(_candidates, _targets, metric_name="bart_score")
        scores.update({'bart_score': bart_scores})
    if 'bart_score_cnn' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        bart_scores = eval_bartscore(_candidates, _targets, metric_name="bart_score_cnn")
        scores.update({'bart_score_cnn': bart_scores})
    if 'bart_score_para' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        bart_scores = eval_bartscore(_candidates, _targets, metric_name="bart_score_para")
        scores.update({'bart_score_para': bart_scores})  
    if 'bart_score_src_hypo' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        bart_scores = eval_bartscore_src_hypo(_candidates, sources, metric_name="bart_score")
        scores.update({'bart_score_src_hypo': bart_scores})
    if 'bart_score_cnn_src_hypo' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        bart_scores = eval_bartscore_src_hypo(_candidates, sources, metric_name="bart_score_cnn")
        scores.update({'bart_score_cnn_src_hypo': bart_scores})
    if 'bart_score_para_src_hypo' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        bart_scores = eval_bartscore_src_hypo(_candidates, sources, metric_name="bart_score_para")
        scores.update({'bart_score_para_src_hypo': bart_scores}) 

    if 'unieval_sum' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        unieval_sum_scores = eval_unieval(_candidates, _targets, sources, task='summarization')
        for aspect in unieval_sum_scores:
            scores.update({f'unieval_sum_{aspect}': unieval_sum_scores[aspect]})
    if 'unieval_dialogue' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        unieval_dislogue_scores = eval_unieval(_candidates, _targets, sources, task='dialogue')
        for aspect in unieval_dislogue_scores:
            scores.update({f'unieval_dialogue_{aspect}': unieval_dislogue_scores[aspect]})
    if 'unieval_fact' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        unieval_fact_scores = eval_unieval(_candidates, _targets, sources, task='fact')
        for aspect in unieval_fact_scores:
            scores.update({f'unieval_fact_{aspect}': unieval_fact_scores[aspect]})
    if 'unieval_d2t' in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        unieval_intermediate_scores = eval_unieval(_candidates, _targets, sources, task='data2text')
        for aspect in unieval_intermediate_scores:
            scores.update({f'unieval_d2t_{aspect}': unieval_intermediate_scores[aspect]})

    if "instructscore" in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        instruct_scores = eval_instructscore(_candidates, _targets)
        scores.update({'instructscore': instruct_scores})
    
    if "gptscore_flan_sum" in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        gpt_scores = eval_gptscore_ref_ist(_candidates, _targets, task="Summ", metric_name="flan_base_score")
        for aspect in gpt_scores:
            scores.update({f'gptscore_flan_sum_{aspect}': gpt_scores[aspect]})
    if "gptscore_flan_d2t" in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        gpt_scores = eval_gptscore_ref_ist(_candidates, _targets, task="D2T", metric_name="flan_base_score")
        for aspect in gpt_scores:
            scores.update({f'gptscore_flan_d2t_{aspect}': gpt_scores[aspect]})
    if "gptscore_flan_dial" in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        gpt_scores = eval_gptscore_ref_ist(_candidates, _targets, task="Dial", metric_name="flan_base_score")
        for aspect in gpt_scores:
            scores.update({f'gptscore_flan_dial_{aspect}': gpt_scores[aspect]})
    if "gptscore_flan_mt" in metrics:   
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        gpt_scores = eval_gptscore_ref_ist(_candidates, _targets, task="MT", metric_name="flan_base_score")
        for aspect in gpt_scores:
            scores.update({f'gptscore_flan_mt_{aspect}': gpt_scores[aspect]})
    
    if "gptscore_flan_sum_src_hypo" in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        gpt_scores = eval_gptscore_src_ist(_candidates, sources, task="Summ", metric_name="flan_base_score")
        for aspect in gpt_scores:
            scores.update({f'gptscore_flan_sum_src_hypo_{aspect}': gpt_scores[aspect]})
    if "gptscore_flan_d2t_src_hypo" in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        gpt_scores = eval_gptscore_src_ist(_candidates, sources, task="D2T", metric_name="flan_base_score")
        for aspect in gpt_scores:
            scores.update({f'gptscore_flan_d2t_src_hypo_{aspect}': gpt_scores[aspect]})
    if "gptscore_flan_dial_src_hypo" in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        gpt_scores = eval_gptscore_src_ist(_candidates, sources, task="Dial", metric_name="flan_base_score")
        for aspect in gpt_scores:
            scores.update({f'gptscore_flan_dial_src_hypo_{aspect}': gpt_scores[aspect]})
    if "gptscore_flan_mt_src_hypo" in metrics:
        _candidates, _targets = deepcopy(candidates), deepcopy(targets)
        gpt_scores = eval_gptscore_src_ist(_candidates, sources, task="MT", metric_name="flan_base_score")
        for aspect in gpt_scores:
            scores.update({f'gptscore_flan_mt_src_hypo_{aspect}': gpt_scores[aspect]})
    
    

    if len(scores) == 0:
        logging.warning("No metric is matched in {}, please check the spelling".format(metrics))  
    if do_flatten:
        for metric in scores:
            assert all([len(score) == 1 for score in scores[metric]])
            scores[metric] = [score[0] for score in scores[metric]]
    return scores

def overall_eval(
    candidates:Union[List[List[str]], List[str]],
    targets: Union[List[str], List[List[str]]],
    metrics:List[str],
    sources:List[str]=None,
    num_workers:int=1,
    ) -> Dict[str, List[float]]:
    """
    Args:
        candidates: the candidates
        targets: the targets
        metrics: the metrics to be evaluated
        num_workers: the number of workers to be used
    Return:
        A dict of scores, same shape with candidates for each metric
    """
    if num_workers > 1:
        cpu_num = psutil.cpu_count(logical=False)
        num_workers = min(num_workers, cpu_num)
        print("Using {} workers to evaluate".format(num_workers))
        chunk_size = len(candidates) // num_workers + 1
        candidates_chunks = [candidates[i:i + chunk_size] for i in range(0, len(candidates), chunk_size)]
        targets_chunks = [targets[i:i + chunk_size] for i in range(0, len(targets), chunk_size)]
        if sources is not None:
            sources_chunks = [sources[i:i + chunk_size] for i in range(0, len(sources), chunk_size)]
            datas = [(candidates_chunks[i], targets_chunks[i], metrics, sources_chunks[i]) for i in range(len(candidates_chunks))]
        else:
            datas = [(candidates_chunks[i], targets_chunks[i], metrics) for i in range(len(candidates_chunks))]
        scores_chunks = process_map(_overall_eval_multi_process, datas, chunksize=1, max_workers=num_workers)
        scores = {}
        for chunk in scores_chunks:
            for k, v in chunk.items():
                scores[k] = scores.get(k, []) + v
    else:
        scores = _overall_eval(candidates, targets, metrics, sources=sources)
    return scores



tigerscore/common/evaluation.py
TIGER-AI-Lab/TIGERScore
# %%
import torch
import torch.nn as nn
import traceback
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer

class FLANScorer:
    def __init__(self, device='cuda:0', max_length=1024, checkpoint='google/flan-t5-base'):
        # Set up model
        self.device = device
        self.max_length = max_length
        self.tokenizer = AutoTokenizer.from_pretrained(checkpoint)
        self.model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)
        self.model.eval()
        self.model.to(device)
        # Set up loss
        self.loss_fct = nn.NLLLoss(reduction='none', ignore_index=self.model.config.pad_token_id)
        self.lsm = nn.LogSoftmax(dim=1)

    def load(self):
        """ Load model from paraphrase finetuning """
        self.model.load_state_dict(torch.load('models/bart.pth', map_location=self.device))

    def score(self, srcs, tgts, batch_size):
        """ Score a batch of examples """
        score_list = []
        for i in range(0, len(srcs), batch_size):
            src_list = srcs[i: i + batch_size]
            tgt_list = tgts[i: i + batch_size]
            if i <1:
                pass
                # print('src_list: ',src_list)
                # print('tgt_list: ', tgt_list)
            try:
                with torch.no_grad():
                    encoded_src = self.tokenizer(
                        src_list,
                        max_length=self.max_length,
                        truncation=True,
                        padding=True,
                        return_tensors='pt'
                    )
                    encoded_tgt = self.tokenizer(
                        tgt_list,
                        max_length=self.max_length,
                        truncation=True,
                        padding=True,
                        return_tensors='pt'
                    )
                    src_tokens = encoded_src['input_ids'].to(self.device)
                    src_mask = encoded_src['attention_mask'].to(self.device)
                    tgt_tokens = encoded_tgt['input_ids'].to(self.device)
                    tgt_mask = encoded_tgt['attention_mask']
                    tgt_len = tgt_mask.sum(dim=1).to(self.device)

                    output = self.model(
                        input_ids=src_tokens,
                        attention_mask=src_mask,
                        labels=tgt_tokens
                    )
                    logits = output.logits.view(-1, self.model.config.vocab_size)
                    loss = self.loss_fct(self.lsm(logits), tgt_tokens.view(-1))
                    loss = loss.view(tgt_tokens.shape[0], -1)
                    loss = loss.sum(dim=1) / tgt_len
                    curr_score_list = [-x.item() for x in loss]
                    score_list += curr_score_list

            except RuntimeError:
                traceback.print_exc()
                print(f'source: {src_list}')
                print(f'target: {tgt_list}')
                exit(0)
        return score_list

tigerscore/common/flan_score.py
TIGER-AI-Lab/TIGERScore
import random
import os
import numpy as np
import torch
import argparse
import hashlib
import requests
import time
from io import BytesIO
from tqdm import tqdm
from PIL import Image
from typing import List, Dict
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from datasets import load_dataset
from datasets.utils.file_utils import get_datasets_user_agent

USER_AGENT = get_datasets_user_agent()

def seed_everything(seed=42):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True

def str2bool(v):
    if isinstance(v, bool):
        return v
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

def empty2None(x):
    if x == '':
        return None
    elif isinstance(x, str):
        return x
    else:
        raise argparse.ArgumentTypeError('String value expected.')

def empty2Noneint(x):
    if x == '':
        return None
    elif isinstance(x, int):
        return x
    elif isinstance(x, str):
        return int(x)
    else:
        raise argparse.ArgumentTypeError('Integer value expected.')

def empty2zero(x):
    if x == '':
        return 0
    elif isinstance(x, int):
        return x
    elif isinstance(x, str):
        return int(x)
    else:
        raise argparse.ArgumentTypeError('Integer value expected.')



def generate_hash_code(text):
    if text is None:
        return None
    # Convert the text to bytes and create a hash object
    hash_object = hashlib.sha256(text.encode())

    # Get the hexadecimal representation of the hash code
    hex_code = hash_object.hexdigest()

    # Return the first 16 digits of the hexadecimal code
    return hex_code[:16]

def fetch_single_image(image_url, timeout=None, retries=2):
    if os.path.exists(image_url):
        # fetch from local
        try:
            image = Image.open(image_url).convert("RGB")
        except Exception as e:
            if retries > 0:
                time.sleep(3)
                return fetch_single_image(image_url, timeout=timeout, retries=retries - 1)
    else:
        # fetch from url
        try:
            r = requests.get(image_url, timeout=timeout, stream=True, headers={"User-Agent": USER_AGENT})
            r.raise_for_status()
            image = Image.open(BytesIO(r.content)).convert("RGB")
        except Exception as e:
            if retries > 0:
                time.sleep(3) # Wait 3 seconds before retrying
                return fetch_single_image(image_url, timeout=timeout, retries=retries - 1)
            else:
                print(f"Failed to fetch image from {image_url} after {retries} retries")
                raise e
    return image

def fetch_images(image_urls, num_threads, timeout=None, retries=2):
    """
    Fetch images from a list of URLs in parallel.
    Args:
        image_urls (list): List of image URLs.
        num_threads (int): Number of threads to use.
        timeout (int, optional): Timeout for the request. Defaults to None.
        retries (int, optional): Number of retries. Defaults to 0.
    Returns:
        list: List of PIL images.
    """
    fetch_single_image_with_args = partial(fetch_single_image, timeout=timeout, retries=retries)
    with ThreadPoolExecutor(max_workers=num_threads) as executor:
        images = list(
            tqdm(
            executor.map(fetch_single_image_with_args, image_urls), 
            total=len(image_urls),
            desc="Fetching images")
        )
    print("Fetched {} images".format(len(images)))
    return images

tigerscore/common/utils.py
TIGER-AI-Lab/TIGERScore
"""
Usage: transforms the xgptscore data format into the alpaca data format for finetuning.

"""
import json
import logging
import sys
import os
import random
import fire
import regex as re
import numpy as np
from collections import Counter
from itertools import chain

sys.path.append("../")
templates_path = os.path.join(os.path.dirname(__file__), "..")
sys.path.append(templates_path)
from string import Template
from common.evaluation import overall_eval
from xgptscore.constants import EVAL_ASPECTS
from pathlib import Path
from common.datasets_config import DATASETS_CONFIG
from transformers import AutoTokenizer
from tqdm import tqdm


# FINETUNE_INST = "You are evaluating errors in a model-generated output for a(an) ${task} task."
# FINETUNE_INPUT = """\
# Task instruction: ${generation_instruction}
# Source: ${input_context}
# Model-generated Output: ${hypothesis_output}

# Based on the given task instruction and source, identify the major and minor errors in this model-generated output.
# Note that Major errors refer to actual errors that affects the task severely, and Minor errors refer to small imperfections, and purely subjective opinions about the output.
# For each error you give in the response, please also elaborate the following information:
# - error location (the words that are wrong in the output)
# - error aspect it belongs to.
# - explanation why it's an error, and the correction suggestions.
# - severity of the error ("Major" or "Minor"). 
# - reduction of score (between 0.5 and 5)

# Your evaluation output in the json format:
# """

FINETUNE_INST = "You are evaluating errors in a model-generated output for a(an) ${task} task."
FINETUNE_INPUT = """\
Task instruction: ${generation_instruction}
Source: ${input_context}
Model-generated Output: ${hypothesis_output}

Based on the given task instruction and source, identify errors in this model-generated output.
For each error you give in the response, please also elaborate the following information:
- error location (the words that are wrong in the output)
- error aspect it belongs to.
- explanation why it's an error, and the correction suggestions.
- severity of the error ("Major" or "Minor"). 
- reduction of score (between 0.5 and 5 given the severity of the error)

Your evaluation output:
"""


def main(
    task: str,
    seed: int = 42,
    input_file: str = None,
    output_file: str = None,
    overwrite: bool = False,
    max_eval_input_length: int = None,
    max_eval_hyp_length: int = None,
    max_eval_output_length: int = None,
):
    if task == "all":
        tasks = list(DATASETS_CONFIG.keys())
    else:
        assert task in DATASETS_CONFIG.keys()
        tasks = [task]

    tokenizer = AutoTokenizer.from_pretrained("huggyllama/llama-7b")

    with open(input_file, 'r') as f:
        data = json.load(f)
    formatted_data = []
    for item in data:
        for cand in item['candidates']:
            inst = Template(FINETUNE_INST).substitute(task=task)
            input_context_ids = tokenizer.encode(item['input'], add_special_tokens=False)
            hyp_ids = tokenizer.encode(cand['text'], add_special_tokens=False)
            if max_eval_input_length is not None and len(input_context_ids) > max_eval_input_length:
                input_context = tokenizer.decode(input_context_ids[:max_eval_input_length]) + "..."
            else:
                input_context = item['input']
            if max_eval_hyp_length is not None and len(hyp_ids) > max_eval_hyp_length:
                hypothesis_output = tokenizer.decode(hyp_ids[:max_eval_hyp_length]) + "..."
            else:
                hypothesis_output = cand['text']
            input_ = Template(FINETUNE_INPUT).substitute(
                generation_instruction=item['instruction'],
                input_context=input_context,
                hypothesis_output=hypothesis_output,
            )
            if isinstance(cand['responses'][-1], dict):
                # important, must not ensure ascii, other wise the outout will be corrupted
                output_ = json.dumps(cand['responses'][-1], ensure_ascii=False, indent=0)
                output_ids = tokenizer.encode(output_, add_special_tokens=False)
                if max_eval_output_length is not None and len(output_ids) > max_eval_output_length:
                    # remove some minor errors to make the output shorter
                    sorted_errors = sorted(cand['responses'][-1]['errors'].values(), key=lambda x: np.abs(x['score_reduction']), reverse=True)
                    while len(output_ids) > max_eval_output_length:
                        removed_error = sorted_errors.pop()
                        # print("removing 1 error with reduction score {} from current {} errors".format(removed_error['score_reduction'], len(sorted_errors)))
                        formated_errors = {"errors": {"error_{}".format(i): error for i, error in enumerate(sorted_errors)}}
                        output_ids = tokenizer.encode(json.dumps(formated_errors, ensure_ascii=False, indent=0), add_special_tokens=False)
                    output_ = json.dumps(formated_errors, ensure_ascii=False, indent=0)
                # transform json to txt
                output_ = json.loads(output_)
                txt_output = "The model-generated output contains {} errors, with a total score reduction of {}.".format(
                    len(output_['errors']), 
                    np.abs(sum([error['score_reduction'] for error in output_['errors'].values()]))
                )
                for i, error in enumerate(output_['errors'].values()):
                    error_txt = ""
                    keys = ["error_location","error_aspect","explanation","severity","score_reduction"]
                    for key in keys:
                        if key == 'type':
                            key = 'error type'
                        error_txt += "{} {}: {}\n".format(
                            key.replace("_", " ").capitalize(),
                            i + 1,
                            error[key],
                        )
                    txt_output += "\n" + error_txt.strip(' \n')
                output_ = txt_output.strip(' \n')

                formatted_data.append({
                    "instruction": inst,
                    "input": input_,
                    "output": output_,
                    "task": task,
                })
    
    with open(output_file, 'w') as f:
        json.dump(formatted_data, f, indent=4, ensure_ascii=False)
    logging.info(f"Saved to {output_file}")
        

    # count the dataset statistics
    dataset_statistics = {}
    dataset_statistics["#total"] = len(formatted_data)
    dataset_statistics["#unique input"] = len(set([item["input"] for item in formatted_data]))
    input_lens = [len(tokenizer.encode(item["input"])) for item in tqdm(formatted_data, desc="Counting input length")]
    output_lens = [len(tokenizer.encode(item["output"])) for item in tqdm(formatted_data, desc="Counting output length")]
    total_lens = [x + y for x, y in zip(input_lens, output_lens)]
    dataset_statistics["input_length"] = {}
    dataset_statistics["input_length"]["mean"] = np.mean(input_lens).item()
    dataset_statistics["input_length"]["percentile"] = np.percentile(input_lens, [0, 25, 50, 90, 100]).tolist()
    dataset_statistics["input_length"]["max"] = max(input_lens)
    dataset_statistics["input_length"]["min"] = min(input_lens)
    dataset_statistics["output_length"] = {}
    dataset_statistics["output_length"]["mean"] = np.mean(output_lens).item()
    dataset_statistics["output_length"]["percentile"] = np.percentile(output_lens, [0, 25, 50, 90, 100]).tolist()
    dataset_statistics["output_length"]["max"] = max(output_lens)
    dataset_statistics["output_length"]["min"] = min(output_lens)
    dataset_statistics["total_length"] = {}
    dataset_statistics["total_length"]["mean"] = np.mean(total_lens).item()
    dataset_statistics["total_length"]["percentile"] = np.percentile(total_lens, [0, 25, 50, 90, 100]).tolist()
    dataset_statistics["total_length"]["max"] = max(total_lens)
    dataset_statistics["total_length"]["min"] = min(total_lens)
    error_aspects = [re.findall(r'(?<=Error aspect \d+: )[ \w]+', item['output']) for item in formatted_data]
    error_aspects = list(chain(*error_aspects))
    dataset_statistics["error_aspects_distribution"] = Counter(error_aspects)
    
    num_errors = [len(re.findall(r'(?<=Error location \d+: ).*(?=\n|$)', item['output'])) for item in formatted_data]
    dataset_statistics["num_errors_distribution"] = Counter(num_errors)
    # severity distributions
    severities = [re.findall(r'(?<=Severity \d+: ).*(?=\n|$)', item['output']) for item in formatted_data]
    severities = list(chain(*severities))
    dataset_statistics["severity_distribution"] = Counter(severities)
    # score reduction distributions
    score_reductions = [re.findall(r'(?<=Score reduction \d+: ).*(?=\n|$)', item['output']) for item in formatted_data]
    score_reductions = list(chain(*score_reductions))
    score_reductions = [abs(float(x.replace(" ", ""))) for x in score_reductions]
    dataset_statistics["score_reduction_distribution"] = Counter(score_reductions)
    
    print(dataset_statistics)
    output_file = Path(output_file).with_suffix(".statistics.json")
    with open(output_file, "w") as f:
        json.dump(dataset_statistics, f, indent=4, ensure_ascii=False)
    logging.info(f"Saved statistics to {output_file}")
        

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    fire.Fire(main)
                        
tigerscore/distill/format_distill_data.py
TIGER-AI-Lab/TIGERScore
"""
Usage: transforms the xgptscore data format into the alpaca data format for finetuning.

"""
import json
import logging
import sys
import os
import random
import fire
import regex as re
import numpy as np
from collections import Counter
from itertools import chain

sys.path.append("../")
templates_path = os.path.join(os.path.dirname(__file__), "..")
sys.path.append(templates_path)
from string import Template
from common.evaluation import overall_eval
from xgptscore.constants import EVAL_ASPECTS
from pathlib import Path
from common.datasets_config import DATASETS_CONFIG
from transformers import AutoTokenizer
from tqdm import tqdm


FINETUNE_INST = "You are evaluating errors in a model-generated output for a(an) ${task} task."
FINETUNE_INPUT = """\
Task instruction: ${generation_instruction}
Source: ${input_context}
Model-generated Output: ${hypothesis_output}

Based on the given task instruction and source, identify errors in this model-generated output.
For each error you give in the response, please also elaborate the following information:
- error location (the words that are wrong in the output)
- error aspect it belongs to.
- explanation why it's an error, and the correction suggestions.
- severity of the error ("Major" or "Minor"). 
- reduction of score (an interger between 1 and 5 given the severity of the error)

Your evaluation output:
"""

def main(
    task: str,
    seed: int = 42,
    input_file: str = None,
    output_file: str = None,
    overwrite: bool = False,
    max_eval_input_length: int = None,
    max_eval_hyp_length: int = None,
    max_eval_output_length: int = None,
):
    if task == "all":
        tasks = list(DATASETS_CONFIG.keys())
    else:
        assert task in DATASETS_CONFIG.keys()
        tasks = [task]

    tokenizer = AutoTokenizer.from_pretrained("huggyllama/llama-7b")

    with open(input_file, 'r') as f:
        data = json.load(f)
    formatted_data = []
    for item in data:
            syn_output = item['responses'][-1]
            syn_output = syn_output.replace(": \n", ": ")
            # decode the synthesis outputs
            try:
                start_pos = syn_output.index("Generated incorrect output: ") + len("Generated incorrect output: ")
                end_pos = syn_output.index("\nError location 1")
                hyp = syn_output[start_pos:end_pos].strip('\n ')
                assert len(hyp) > 0
            except Exception as e:
                logging.warning("Failed to parse the synthesis output: {}".format(syn_output))
                continue
            inst = Template(FINETUNE_INST).substitute(task=task)
            input_context_ids = tokenizer.encode(item['input'], add_special_tokens=False)
            hyp_ids = tokenizer.encode(hyp, add_special_tokens=False)
            if max_eval_input_length is not None and len(input_context_ids) > max_eval_input_length:
                input_context = tokenizer.decode(input_context_ids[:max_eval_input_length]) + "..."
            else:
                input_context = item['input']
            if max_eval_hyp_length is not None and len(hyp_ids) > max_eval_hyp_length:
                hypothesis_output = tokenizer.decode(hyp_ids[:max_eval_hyp_length]) + "..."
            else:
                hypothesis_output = hyp
            input_ = Template(FINETUNE_INPUT).substitute(
                generation_instruction=item['instruction'],
                input_context=input_context,
                hypothesis_output=hypothesis_output,
            )
            try:
                error_locations = re.findall(r'(?<=Error location \d+: ).*(?=\n|$)', syn_output)
                error_aspects = re.findall(r'(?<=Error aspect \d+: ).*(?=\n|$)', syn_output)
                explanations = re.findall(r'(?<=Explanation \d+: ).*(?=\n|$)', syn_output)
                severities = re.findall(r'(?<=Severity \d+: ).*(?=\n|$)', syn_output)
                score_reductions = re.findall(r'(?<=Score reduction \d+: ).*(?=\n|$)', syn_output)
                score_reductions = [abs(int(x.replace(" ", ""))) for x in score_reductions]
            except Exception as e:
                logging.warning("Failed to parse the synthesis output: {}".format(syn_output))
                continue
            
            if not len(error_locations) == len(error_aspects) == len(explanations) == len(severities) == len(score_reductions):
                logging.warning("The number of errors properties does not match!: {}".format(syn_output))
                continue
            
            txt_output = "The model-generated output contains {} errors, with a total score reduction of {}.".format(
                len(error_locations),
                sum([int(score) for score in score_reductions]),
            )
            task_eval_aspects = list(EVAL_ASPECTS[task].keys())
            for i in range(len(error_locations)):
                txt_output += "\nError location {}: {}\n".format(i + 1, error_locations[i])
                txt_output += "Error aspect {}: {}\n".format(i + 1, error_aspects[i])
                txt_output += "Explanation {}: {}\n".format(i + 1, explanations[i])
                txt_output += "Severity {}: {}\n".format(i + 1, severities[i])
                txt_output += "Score reduction {}: {}".format(i + 1, score_reductions[i])
            output_ = txt_output.strip(' \n')
            formatted_data.append({
                "instruction": inst,
                "input": input_,
                "output": output_,
                "task": task,
            })
    
    # # append 20% non-error examples
    # for item in data:
    #     if random.random() < 0.2:
    #         inst = Template(FINETUNE_INST).substitute(task=task)
    #         input_context_ids = tokenizer.encode(item['input'], add_special_tokens=False)
    #         if max_eval_input_length is not None and len(input_context_ids) > max_eval_input_length:
    #             input_context = tokenizer.decode(input_context_ids[:max_eval_input_length]) + "..."
    #         else:
    #             input_context = item['input']
    #         input_ = Template(FINETUNE_INPUT).substitute(
    #             generation_instruction=item['instruction'],
    #             input_context=input_context,
    #             hypothesis_output=item['output'],
    #         )
    #         output_ = "The model-generated output contains 0 errors, with a total score reduction of 0."
    #         formatted_data.append({
    #             "instruction": inst,
    #             "input": input_,
    #             "output": output_,
    #             "task": task,
    #         })
    
    with open(output_file, 'w') as f:
        json.dump(formatted_data, f, indent=4, ensure_ascii=False)
    logging.info(f"Saved to {output_file}")
        

    # count the dataset statistics
    dataset_statistics = {}
    dataset_statistics["#total"] = len(formatted_data)
    dataset_statistics["#unique input"] = len(set([item["input"] for item in formatted_data]))
    input_lens = [len(tokenizer.encode(item["input"])) for item in tqdm(formatted_data, desc="Counting input length")]
    output_lens = [len(tokenizer.encode(item["output"])) for item in tqdm(formatted_data, desc="Counting output length")]
    total_lens = [x + y for x, y in zip(input_lens, output_lens)]
    dataset_statistics["input_length"] = {}
    dataset_statistics["input_length"]["mean"] = np.mean(input_lens).item()
    dataset_statistics["input_length"]["percentile"] = np.percentile(input_lens, [0, 25, 50, 90, 100]).tolist()
    dataset_statistics["input_length"]["max"] = max(input_lens)
    dataset_statistics["input_length"]["min"] = min(input_lens)
    dataset_statistics["output_length"] = {}
    dataset_statistics["output_length"]["mean"] = np.mean(output_lens).item()
    dataset_statistics["output_length"]["percentile"] = np.percentile(output_lens, [0, 25, 50, 90, 100]).tolist()
    dataset_statistics["output_length"]["max"] = max(output_lens)
    dataset_statistics["output_length"]["min"] = min(output_lens)
    dataset_statistics["total_length"] = {}
    dataset_statistics["total_length"]["mean"] = np.mean(total_lens).item()
    dataset_statistics["total_length"]["percentile"] = np.percentile(total_lens, [0, 25, 50, 90, 100]).tolist()
    dataset_statistics["total_length"]["max"] = max(total_lens)
    dataset_statistics["total_length"]["min"] = min(total_lens)
    error_aspects = [re.findall(r'(?<=Error aspect \d+: ).*(?=\n|$)', item['output']) for item in formatted_data]
    error_aspects = list(chain(*error_aspects))
    dataset_statistics["error_aspects_distribution"] = Counter(error_aspects)
    # number of errors distributions
    num_errors = [len(re.findall(r'(?<=Error location \d+: ).*(?=\n|$)', item['output'])) for item in formatted_data]
    dataset_statistics["num_errors_distribution"] = Counter(num_errors)
    # severity distributions
    severities = [re.findall(r'(?<=Severity \d+: ).*(?=\n|$)', item['output']) for item in formatted_data]
    severities = list(chain(*severities))
    dataset_statistics["severity_distribution"] = Counter(severities)
    # score reduction distributions
    score_reductions = [re.findall(r'(?<=Score reduction \d+: ).*(?=\n|$)', item['output']) for item in formatted_data]
    score_reductions = list(chain(*score_reductions))
    score_reductions = [abs(int(x.replace(" ", ""))) for x in score_reductions]
    dataset_statistics["score_reduction_distribution"] = Counter(score_reductions)
    
    print(dataset_statistics)
    output_file = Path(output_file).with_suffix(".statistics.json")
    with open(output_file, "w") as f:
        json.dump(dataset_statistics, f, indent=4, ensure_ascii=False)
    logging.info(f"Saved statistics to {output_file}")
        

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    fire.Fire(main)
                        
tigerscore/distill/format_synthesis_distill_data.py
TIGER-AI-Lab/TIGERScore
import json
import transformers
import torch
import logging
import sys
import regex as re
from pathlib import Path
from tqdm import tqdm
from dataclasses import dataclass, field
from typing import Optional, Sequence, Dict, List
from torch.utils.data import Dataset, DataLoader
from string import Template
from mt_metrics_eval.stats import Correlation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.process_utils import get_xgptscore_from_json, json_postprocess
from peft import PeftModel

IGNORE_INDEX = -100
DEFAULT_PAD_TOKEN = "[PAD]"
DEFAULT_EOS_TOKEN = "</s>"
DEFAULT_BOS_TOKEN = "<s>"
DEFAULT_UNK_TOKEN = "<unk>"
PROMPT_DICT = {
    "prompt_input": (
        "Below is an instruction that describes a task, paired with an input that provides further context. "
        "Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:"
    ),
    "prompt_no_input": (
        "Below is an instruction that describes a task. "
        "Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Response:"
    ),
}
# FINETUNE_INST = "You are evaluating the errors in a model-generated output for a(an) ${task} task."
# FINETUNE_INPUT = """\
# Task instruction: ${generation_instruction}
# Source: ${input_context}
# Model-generated Output: ${hypothesis_output}

# Based on the given task instruction and source, identify the major and minor errors in this model-generated output.
# Note that Major errors refer to actual errors that affects the task severely, and Minor errors refer to small imperfections, and purely subjective opinions about the output.
# For each error you give in the response, please also elaborate the following information:
# - error location (the words that are wrong in the output)
# - error aspect it belongs to.
# - explanation why it's an error, and the correction suggestions.
# - severity of the error ("Major" or "Minor"). 
# - reduction of score (between 0.5 and 5)

# Your evaluation output in the json format:
# """

# FINETUNE_INST = ""
# FINETUNE_INPUT = """\
# Task instruction: ${generation_instruction}
# Source: ${input_context}
# Output: ${hypothesis_output}
# Based on the given task instruction and source, identify the major and minor errors in this output for a(an) ${task} task. Note that Major errors refer to actual errors that affects the task severely, and Minor errors refer to smaller imperfections, and purely subjective opinions about the output.
# For each error you give in the response, please also elaborate the following information:
# - error location (the words that are wrong in the output)
# - error aspect it belongs to.
# - explanation why it's an error, and the correction suggestions.
# - severity of the error ("Major" or "Minor"). 
# - reduction of score (Major for 5, Minor for 1)
# Your evaluation output in the json format:
# """



@dataclass
class ModelArguments:
    model_name_or_path: Optional[str] = field(
        default="facebook/opt-125m", metadata={"help": "Path to the model checkpoint or pretrained model."})
    load_lora: str = field(default=None, metadata={"help": "Path to the lora model checkpoint."})

@dataclass
class DataArguments:
    task: str = field(default="translation", metadata={"help": "Task to evaluate on."})
    max_eval_input_length: int = field(
        default=256,
        metadata={"help": "Maximum input context (e.g. source text for translation) sequence length."},
    )
    max_eval_hyp_length: int = field(
        default=128,
        metadata={"help": "Maximum hypothesis output (e.g. candidate text for translation) sequence length."},
    )
    max_eval_output_length: int = field(
        default=256,
        metadata={"help": "Maximum output (e.g. evaluation results) sequence length."},
    )
    data_path: str = field(default=None, metadata={"help": "Path to the training data."})
    output_path: str = field(default=None, metadata={"help": "Path to the output file."})
    human_score_names: str = field(default="mqm,da", metadata={"help": "Name of the human scores."})
    overwrite: bool = field(default=False, metadata={"help": "Overwrite the output file if it exists."})

@dataclass
class EvalArguments:
    batch_size: int = field(default=8, metadata={"help": "Batch size for evaluation."})
    cache_dir: Optional[str] = field(default=None)
    model_max_length: int = field(default=512, metadata={"help": "Maximum sequence length."})
    torch_dtype: str = field(default="float32", metadata={"help": "Data type to use for evaluation."})
    decoding_method: Literal["beam_search", "top_p_sampling"] = field(
        default="top_p_sampling", metadata={"help": "Decoding method."}
    )
def smart_tokenizer_and_embedding_resize(
    special_tokens_dict: Dict,
    tokenizer: transformers.PreTrainedTokenizer,
    model: transformers.PreTrainedModel,
):
    """Resize tokenizer and embedding.

    Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
    """
    num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)
    model.resize_token_embeddings(len(tokenizer))

    if num_new_tokens > 0:
        input_embeddings = model.get_input_embeddings().weight.data
        output_embeddings = model.get_output_embeddings().weight.data

        input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)
        output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)

        input_embeddings[-num_new_tokens:] = input_embeddings_avg
        output_embeddings[-num_new_tokens:] = output_embeddings_avg


class EvalDataset(Dataset):
    """Dataset for supervised fine-tuning."""

    def __init__(self, data_args: DataArguments, tokenizer: transformers.PreTrainedTokenizer):
        super(EvalDataset, self).__init__()
        logging.info("Loading data...")
        with open(data_args.data_path, "r") as f:
            self.examples = json.load(f)
        logging.info("Formatting inputs...")
        formatted_data = []
        for item in self.examples:
            for cand in item['candidates']:
                inst = Template(FINETUNE_INST).substitute(task=data_args.task)
                input_ = Template(FINETUNE_INPUT).substitute(
                    task=data_args.task,
                    generation_instruction=item['instruction'],
                    input_context=item['input'],
                    hypothesis_output=cand['text'],
                )
                formatted_data.append({
                    "instruction": inst,
                    "input": input_,
                })
        # for item in self.examples:
        #     inst = Template(FINETUNE_INST).substitute(task=data_args.task)
        #     input_ = Template(FINETUNE_INPUT).substitute(
        #         task=data_args.task,
        #         generation_instruction=item['instruction'],
        #         input_context=item['input'],
        #         hypothesis_output=item['output'][len("Generated incorrect output: "):item['output'].find("\n")],
        #     )
        #     formatted_data.append({
        #         "instruction": inst,
        #         "input": input_,
        #     })
        self.formated_data = formatted_data
        sources = [example['instruction'] + '\n' + example['input'] for example in formatted_data]
        sources = [x.strip(' \n') + "\n" for x in sources]
        logging.info("Tokenizing inputs... This may take some time...")
        self.input_ids = _tokenize_fn(sources, tokenizer)["input_ids"]

    def __len__(self):
        return len(self.input_ids)

    def __getitem__(self, i) -> Dict[str, torch.Tensor]:
        return dict(input_ids=self.input_ids[i], labels=None)


@dataclass
class DataCollatorForEvalDataset(object):
    """Collate examples for supervised fine-tuning."""

    tokenizer: transformers.PreTrainedTokenizer

    def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
        input_ids = [instance["input_ids"] for instance in instances]
        reverse_input_ids = [torch.flip(input_id, dims=(0,)) for input_id in input_ids]
        input_ids = torch.nn.utils.rnn.pad_sequence(
            reverse_input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id
        )
        input_ids = torch.flip(input_ids, dims=(1,))
        return dict(
            input_ids=input_ids,
            labels=None,
            attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
        )

def make_eval_data_module(tokenizer: transformers.PreTrainedTokenizer, data_args) -> Dict:
    """Make dataset and collator for supervised fine-tuning."""
    eval_dataset = EvalDataset(data_args=data_args, tokenizer=tokenizer)
    data_collator = DataCollatorForEvalDataset(tokenizer=tokenizer)
    return dict(train_dataset=None, eval_dataset=eval_dataset, data_collator=data_collator)



def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer) -> Dict:
    """Tokenize a list of strings."""
    tokenized_list = [
        tokenizer(
            text,
            return_tensors="pt",
            padding="longest",
            max_length=tokenizer.model_max_length,
            truncation=True,
        )
        for text in strings
    ]
    input_ids = labels = [tokenized.input_ids[0] for tokenized in tokenized_list]
    input_ids_lens = labels_lens = [
        tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item() for tokenized in tokenized_list
    ]
    return dict(
        input_ids=input_ids,
        labels=labels,
        input_ids_lens=input_ids_lens,
        labels_lens=labels_lens,
    )

def get_sum_penalties(eval_output: dict):
    """
    Args:
        eval_output: dict, the json output of the eval function
    
    Returns:
    """
    try:
        penalty_score = 0
        for aspect in eval_output:
            for penalty_point in eval_output[aspect]["penalty_points"]:
                penalty_score += penalty_point["score_reduction"]
        return - penalty_score
    except:
        return None

def get_torch_dtype(dtype_str):
    """
        Get the torch dtype from a string
    """
    if dtype_str == "float32":
        return torch.float32
    elif dtype_str == "float16":
        return torch.float16
    elif dtype_str == "bfloat16":
        return torch.bfloat16
    elif dtype_str == "int8":
        return torch.int8
    else:
        raise ValueError("Invalid dtype {}".format(dtype_str))


class MyCorrelation(Correlation):
    def __init__(self, num_sys:int, gold_scores:List[int], metric_scores:List[int]):
        # remove nan in metrics scores
        none_metric_scores_idxs = [idx for idx, x in enumerate(metric_scores) if x is None]
        logging.info("Remove {} nan scores from {} scores".format(
            len(none_metric_scores_idxs),
            len(metric_scores)
        ))
        gold_scores = gold_scores.copy()
        # set gold scores to None if metric scores are None
        for idx in none_metric_scores_idxs[::-1]:
            gold_scores[idx] = None 
        super().__init__(num_sys, gold_scores, metric_scores)
        
def main(data_args, model_args, eval_args):

    if data_args.output_path is not None:
        output_file = Path(data_args.output_path)
    else:
        output_file = Path(data_args.data_path).with_suffix('.xgptscore.output.json')
    if not output_file.exists() or data_args.overwrite:
        logging.info("Loading model...")
        model = transformers.AutoModelForCausalLM.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=eval_args.cache_dir,
            torch_dtype=get_torch_dtype(eval_args.torch_dtype),
            device_map="auto"
        )
        logging.info("Model loaded from {}".format(model_args.model_name_or_path))
        model.eval()
        if model_args.load_lora is not None:
            logging.info("Loading lora model...")
            model = PeftModel.from_pretrained(model, model_args.load_lora)
            model = model.merge_and_unload()
            logging.info("Loaded lora model from {}".format(model_args.load_lora))

        tokenizer = transformers.AutoTokenizer.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=eval_args.cache_dir,
            model_max_length=eval_args.model_max_length,
            padding_side="left",
            use_fast=False,
        )
        special_tokens_dict = dict()
        if tokenizer.pad_token is None:
            special_tokens_dict["pad_token"] = DEFAULT_PAD_TOKEN
        if tokenizer.eos_token is None:
            special_tokens_dict["eos_token"] = DEFAULT_EOS_TOKEN
        if tokenizer.bos_token is None:
            special_tokens_dict["bos_token"] = DEFAULT_BOS_TOKEN
        if tokenizer.unk_token is None:
            special_tokens_dict["unk_token"] = DEFAULT_UNK_TOKEN
        
        smart_tokenizer_and_embedding_resize(
            special_tokens_dict=special_tokens_dict,
            tokenizer=tokenizer,
            model=model,
        )

        data_module = make_eval_data_module(tokenizer=tokenizer, data_args=data_args)
        dataloader = DataLoader(
            data_module["eval_dataset"], 
            batch_size=eval_args.batch_size, 
            collate_fn=data_module["data_collator"])

        generation_params = {
            "max_new_tokens": data_args.max_eval_output_length,
            "early_stopping": True,
        }
        if eval_args.decoding_method == "beam_search":
            generation_params["num_beams"] = 4
            logging.info("Using beam search with num_beams = {}".format(generation_params["num_beams"]))
        elif eval_args.decoding_method == "top_p_sampling":
            generation_params["do_sample"] = True
            generation_params["top_p"] = 1.0
            generation_params["temperature"] = 0.7
            logging.info("Using top-p sampling with top_p = {}, temperature = {}".format(
                generation_params["top_p"], generation_params["temperature"]))
        else:
            raise ValueError("Invalid decoding method {}".format(eval_args.decocoding_method))
        
        logging.info("Batch size: {}".format(eval_args.batch_size))
        eval_outputs = []
        with torch.no_grad():
            for batch in tqdm(dataloader, desc="Evaluating", total=len(dataloader)):
                input_ids = batch["input_ids"].to(model.device)
                attention_mask = batch["attention_mask"].to(model.device)
                outputs = model.generate(
                    input_ids=input_ids, 
                    attention_mask=attention_mask,
                    max_new_tokens=data_args.max_eval_output_length,
                    do_sample=True,
                    top_p=0.95,
                    temperature=0.8,
                )
                input_len = input_ids.shape[1]
                outputs = outputs[:, input_len:]
                decoded_outputs = tokenizer.batch_decode(outputs, skip_special_tokens=True)
                eval_outputs.extend(decoded_outputs)

        eval_examples = data_module["eval_dataset"].examples
        cand_idx = 0
        for ex in eval_examples:
            for cand in ex['candidates']:
                # cand['eval_output'] = json_postprocess(eval_outputs[cand_idx])
                # cand['xgptscore'] = get_xgptscore_from_json(cand['eval_output'])
                # if cand['xgptscore'] is None:
                #     cand['xgptscore'] = get_sum_penalties(cand['eval_output'])
                cand['eval_output'] = eval_outputs[cand_idx]
                score_reductions = re.findall(r"(?<=\nScore reduction \d+: )(\d+\.\d+|\d+)", eval_outputs[cand_idx])
                cand['xgptscore'] = -sum(map(float, score_reductions))
                cand_idx += 1
        # for ex in eval_examples:
        #     # cand['eval_output'] = json_postprocess(eval_outputs[cand_idx])
        #     # cand['xgptscore'] = get_xgptscore_from_json(cand['eval_output'])
        #     # if cand['xgptscore'] is None:
        #     #     cand['xgptscore'] = get_sum_penalties(cand['eval_output'])
        #     ex['eval_output'] = eval_outputs[cand_idx]
        #     score_reductions = re.findall(r"(?<=\nScore reduction \d+: )(\d+\.\d+|\d+)", eval_outputs[cand_idx])
        #     ex['xgptscore'] = -sum(map(float, score_reductions))
        #     cand_idx += 1
        
        with open(output_file, 'w') as f:
            json.dump(eval_examples, f, indent=4, ensure_ascii=False)
        logging.info("Saved eval results to {}".format(output_file))
    else:
        with open(output_file, 'r') as f:
            eval_examples = json.load(f)
        for ex in eval_examples:
            for cand in ex['candidates']:
                score_reductions = re.findall(r"(?<=\nScore reduction \d+: )(\d+\.\d+|\d+)", cand['eval_output'])
                cand['xgptscore'] = -sum(map(float, score_reductions))
        with open(output_file, 'w') as f:
            json.dump(eval_examples, f, indent=4, ensure_ascii=False)
        logging.info("Loaded eval results from {}".format(output_file))
    # Compute correlation
    human_score_names = data_args.human_score_names.split(',')
    
    
    for h_name in human_score_names:
        human_scores = []
        xgptscores = []
        for item in eval_examples:
            for cand in item['candidates']:
                flag = False
                for s_name, score in cand['scores'].items():
                        if s_name == h_name:
                            xgptscores.append(cand['xgptscore'])
                            human_scores.append(score)
                            flag = True
                            break
                # assert flag, "No human score found in {}".format(cand['scores'])
        corr = MyCorrelation(1, human_scores, xgptscores)
        logging.info("Human score: {}".format(h_name))
        logging.info("Pearson correlation: {}".format(corr.Pearson()))
        logging.info("Spearman correlation: {}".format(corr.Spearman()))
        logging.info("Kendall correlation: {}".format(corr.Kendall()))

    
    # for item in eval_examples:
    #     for cand in item['candidates']:
    #         flag = False
    #         for s_name, score in cand['scores'].items():
    #             if s_name in human_score_names:
    #                 human_scores.append(score)
    #                 flag = True
    #                 break
    #         assert flag, "No human score found in {}".format(cand['scores'])
    #         xgptscores.append(cand['xgptscore'])
    # corr = MyCorrelation(1, human_scores, xgptscores)
    # logging.info("Pearson correlation: {}".format(corr.Pearson()))
    # logging.info("Spearman correlation: {}".format(corr.Spearman()))
    # logging.info("Kendall correlation: {}".format(corr.Kendall()))


if __name__ == "__main__":
    parser = transformers.HfArgumentParser((ModelArguments, DataArguments, EvalArguments))
    model_args, data_args, eval_args = parser.parse_args_into_dataclasses()

    logging.basicConfig(level=logging.INFO)
    main(data_args, model_args, eval_args)


tigerscore/distill/test_llama.py
TIGER-AI-Lab/TIGERScore
import json
import transformers
import torch
import logging
import sys
from pathlib import Path
from tqdm import tqdm
from dataclasses import dataclass, field
from typing import Optional, Sequence, Dict, List
from torch.utils.data import Dataset, DataLoader
from string import Template
from mt_metrics_eval.stats import Correlation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.process_utils import get_xgptscore_from_json_per_aspect, json_postprocess
from peft import PeftModel
from xgptscore.constants import EVAL_ASPECTS
from eval_scripts.bs_utils import reformat_sum_for_bartscore

IGNORE_INDEX = -100
DEFAULT_PAD_TOKEN = "[PAD]"
DEFAULT_EOS_TOKEN = "</s>"
DEFAULT_BOS_TOKEN = "<s>"
DEFAULT_UNK_TOKEN = "<unk>"
PROMPT_DICT = {
    "prompt_input": (
        "Below is an instruction that describes a task, paired with an input that provides further context. "
        "Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:"
    ),
    "prompt_no_input": (
        "Below is an instruction that describes a task. "
        "Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Response:"
    ),
}
FINETUNE_INST = "You are evaluating the errors in a model-generated output for a(an) ${task} task."
FINETUNE_INPUT = """\
Task instruction: ${generation_instruction}
Source: ${input_context}
Model-generated Output: ${hypothesis_output}

Based on the given task instruction and source, identify the major and minor errors in this model-generated output.
Note that Major errors refer to actual errors that affects the task severely, and Minor errors refer to small imperfections, and purely subjective opinions about the output.
For each error you give in the response, please also elaborate the following information:
- error location (the words that are wrong in the output)
- error aspect it belongs to.
- explanation why it's an error, and the correction suggestions.
- severity of the error ("Major" or "Minor"). 
- reduction of score (between 0.5 and 5)

Your evaluation output in the json format:
"""

@dataclass
class ModelArguments:
    model_name_or_path: Optional[str] = field(
        default="facebook/opt-125m", metadata={"help": "Path to the model checkpoint or pretrained model."})
    load_lora: str = field(default=None, metadata={"help": "Path to the lora model checkpoint."})

@dataclass
class DataArguments:
    task: str = field(default="translation", metadata={"help": "Task to evaluate on."})
    max_eval_input_length: int = field(
        default=256,
        metadata={"help": "Maximum input context (e.g. source text for translation) sequence length."},
    )
    max_eval_hyp_length: int = field(
        default=128,
        metadata={"help": "Maximum hypothesis output (e.g. candidate text for translation) sequence length."},
    )
    max_eval_output_length: int = field(
        default=256,
        metadata={"help": "Maximum output (e.g. evaluation results) sequence length."},
    )
    data_path: str = field(default=None, metadata={"help": "Path to the training data."})
    output_path: str = field(default=None, metadata={"help": "Path to the output file."})
    human_score_names: str = field(default="mqm,da", metadata={"help": "Name of the human scores."})

@dataclass
class EvalArguments:
    batch_size: int = field(default=8, metadata={"help": "Batch size for evaluation."})
    cache_dir: Optional[str] = field(default=None)
    model_max_length: int = field(default=512, metadata={"help": "Maximum sequence length."})
    torch_dtype: str = field(default="float32", metadata={"help": "Data type to use for evaluation."})

def smart_tokenizer_and_embedding_resize(
    special_tokens_dict: Dict,
    tokenizer: transformers.PreTrainedTokenizer,
    model: transformers.PreTrainedModel,
):
    """Resize tokenizer and embedding.

    Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
    """
    num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)
    model.resize_token_embeddings(len(tokenizer))

    if num_new_tokens > 0:
        input_embeddings = model.get_input_embeddings().weight.data
        output_embeddings = model.get_output_embeddings().weight.data

        input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)
        output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)

        input_embeddings[-num_new_tokens:] = input_embeddings_avg
        output_embeddings[-num_new_tokens:] = output_embeddings_avg


class EvalDataset(Dataset):
    """Dataset for supervised fine-tuning."""

    def __init__(self, data_args: DataArguments, tokenizer: transformers.PreTrainedTokenizer):
        super(EvalDataset, self).__init__()
        logging.info("Loading data...")
        with open(data_args.data_path, "r") as f:
            self.examples = json.load(f)[:30] # debug

        logging.info("Formatting inputs...")
        formatted_data = []
        for item in self.examples:
            for cand in item['candidates']:
                inst = Template(FINETUNE_INST).substitute(task=item['task'])
                input_ = Template(FINETUNE_INPUT).substitute(
                    generation_instruction=item['instruction'],
                    input_context=item['input'],
                    hypothesis_output=cand['text'],
                )
                formatted_data.append({
                    "instruction": inst,
                    "input": input_,
                })
        self.formated_data = formatted_data
        sources = [example['instruction'] + '\n' + example['input'] for example in formatted_data]
        sources = [x.lstrip(' \n') for x in sources]
        logging.info("Tokenizing inputs... This may take some time...")
        self.input_ids = _tokenize_fn(sources, tokenizer)["input_ids"]

    def __len__(self):
        return len(self.input_ids)

    def __getitem__(self, i) -> Dict[str, torch.Tensor]:
        return dict(input_ids=self.input_ids[i], labels=None)


@dataclass
class DataCollatorForEvalDataset(object):
    """Collate examples for supervised fine-tuning."""

    tokenizer: transformers.PreTrainedTokenizer

    def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
        input_ids = [instance["input_ids"] for instance in instances]
        reverse_input_ids = [torch.flip(input_id, dims=(0,)) for input_id in input_ids]
        input_ids = torch.nn.utils.rnn.pad_sequence(
            reverse_input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id
        )
        input_ids = torch.flip(input_ids, dims=(1,))
        
        return dict(
            input_ids=input_ids,
            labels=None,
            attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
        )

def make_eval_data_module(tokenizer: transformers.PreTrainedTokenizer, data_args) -> Dict:
    """Make dataset and collator for supervised fine-tuning."""
    eval_dataset = EvalDataset(data_args=data_args, tokenizer=tokenizer)
    data_collator = DataCollatorForEvalDataset(tokenizer=tokenizer)
    return dict(train_dataset=None, eval_dataset=eval_dataset, data_collator=data_collator)



def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer) -> Dict:
    """Tokenize a list of strings."""
    tokenized_list = [
        tokenizer(
            text,
            return_tensors="pt",
            padding="longest",
            max_length=tokenizer.model_max_length,
            truncation=True,
        )
        for text in strings
    ]
    input_ids = labels = [tokenized.input_ids[0] for tokenized in tokenized_list]
    input_ids_lens = labels_lens = [
        tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item() for tokenized in tokenized_list
    ]
    return dict(
        input_ids=input_ids,
        labels=labels,
        input_ids_lens=input_ids_lens,
        labels_lens=labels_lens,
    )

def get_sum_penalties(eval_output: dict):
    """
    Args:
        eval_output: dict, the json output of the eval function
    
    Returns:
    """
    try:
        penalty_score = 0
        for aspect in eval_output:
            for penalty_point in eval_output[aspect]["penalty_points"]:
                penalty_score += penalty_point["score_reduction"]
        return - penalty_score
    except:
        return None

def get_torch_dtype(dtype_str):
    """
        Get the torch dtype from a string
    """
    if dtype_str == "float32":
        return torch.float32
    elif dtype_str == "float16":
        return torch.float16
    elif dtype_str == "bfloat16":
        return torch.bfloat16
    elif dtype_str == "int8":
        return torch.int8
    else:
        raise ValueError("Invalid dtype {}".format(dtype_str))



def main(data_args, model_args, eval_args):
    
    logging.info("Loading model...")
    model = transformers.AutoModelForCausalLM.from_pretrained(
        model_args.model_name_or_path,
        cache_dir=eval_args.cache_dir,
        torch_dtype=get_torch_dtype(eval_args.torch_dtype),
        device_map="auto"
    )
    logging.info("Model loaded from {}".format(model_args.model_name_or_path))
    model.eval()
    if model_args.load_lora is not None:
        logging.info("Loading lora model...")
        model = PeftModel.from_pretrained(model, model_args.load_lora)
        model = model.merge_and_unload()
        logging.info("Loaded lora model from {}".format(model_args.load_lora))

    tokenizer = transformers.AutoTokenizer.from_pretrained(
        model_args.model_name_or_path,
        cache_dir=eval_args.cache_dir,
        model_max_length=eval_args.model_max_length,
        padding_side="left",
        use_fast=False,
    )
    special_tokens_dict = dict()
    if tokenizer.pad_token is None:
        special_tokens_dict["pad_token"] = DEFAULT_PAD_TOKEN
    if tokenizer.eos_token is None:
        special_tokens_dict["eos_token"] = DEFAULT_EOS_TOKEN
    if tokenizer.bos_token is None:
        special_tokens_dict["bos_token"] = DEFAULT_BOS_TOKEN
    if tokenizer.unk_token is None:
        special_tokens_dict["unk_token"] = DEFAULT_UNK_TOKEN
    
    smart_tokenizer_and_embedding_resize(
        special_tokens_dict=special_tokens_dict,
        tokenizer=tokenizer,
        model=model,
    )

    data_module = make_eval_data_module(tokenizer=tokenizer, data_args=data_args)
    dataloader = DataLoader(
        data_module["eval_dataset"], 
        batch_size=eval_args.batch_size, 
        collate_fn=data_module["data_collator"])

    eval_outputs = []
    with torch.no_grad():
        for batch in tqdm(dataloader, desc="Evaluating", total=len(dataloader)):
            input_ids = batch["input_ids"].to(model.device)
            attention_mask = batch["attention_mask"].to(model.device)
            outputs = model.generate(
                input_ids=input_ids, 
                attention_mask=attention_mask,
                max_new_tokens=data_args.max_eval_output_length,
                do_sample=True,
                top_p=1.0,
                temperature=0.7,
            )
            input_len = input_ids.shape[1]
            outputs = outputs[:, input_len:]
            decoded_outputs = tokenizer.batch_decode(outputs, skip_special_tokens=True)
            eval_outputs.extend(decoded_outputs)

    eval_examples = data_module["eval_dataset"].examples
    cand_idx = 0
    aspects = EVAL_ASPECTS["summarization"].keys()
    score_dict = {"xgptscore_"+aspect: 0 for aspect in aspects}
    for ex in eval_examples:
        for cand in ex['candidates']: 
            cand['eval_output'] = json_postprocess(eval_outputs[cand_idx])
            xgptscore_ans = get_xgptscore_from_json_per_aspect(cand['eval_output'])
            if xgptscore_ans is None:
                cand['xgptscore'] = get_sum_penalties(cand['eval_output'])
            else:
                cand['xgptscore'] = score_dict.update(xgptscore_ans)
            cand_idx += 1
    if data_args.output_path is not None:
        output_file = Path(data_args.output_path)
    else:
        output_file = Path(data_args.data_path).with_suffix('.xgptscore.output.json')
    with open(output_file, 'w') as f:
        json.dump(eval_examples, f, indent=4, ensure_ascii=False)
    logging.info("Saved eval results to {}".format(output_file))
    # Compute correlation
    human_score_names = data_args.human_score_names.split(',')
    human_scores = []
    xgptscores = []
    for item in eval_examples:
        for cand in item['candidates']:
            flag = False
            for s_name, score in cand['scores'].items():
                if s_name in human_score_names:
                    human_scores.append(score)
                    flag = True
                    break
            assert flag, "No human score found in {}".format(cand['scores'])
            xgptscores.append(cand['xgptscore'])
    sum_eval_path = Path(f"{output_file}")
    reformat_sum_for_bartscore(sum_eval_path,"xgptscore")
    from eval_scripts.bs_analysis import SUMStat
    summ_stat = SUMStat(str(sum_eval_path.with_name(sum_eval_path.stem + "_bs_format.pkl")))
    # Save correlation results
    corr_results_file = Path("./eval_results/")  / (output_file.stem + ".bs_corr.json")
    corr_results_file.parent.mkdir(parents=True, exist_ok=True)
    
    metrics = [
    'rouge1_r',
    'rouge2_r',
    'rougel_r',
    'bert_score_r',
    'mover_score',
    'prism_hypo_ref',
    "prism_src_hypo",
    'bart_score_cnn_hypo_ref',
    "bart_score_src_hypo",
    'bart_score_para_src_hypo',
    'xgptscore',
    'xgptscore_Fluency',
    'xgptscore_Relevance',
    'xgptscore_Coherence',
    'xgptscore_Consistency',
    ]
    
    summ_stat.evaluate_summary("xgptscore",metrics,corr_results_file)


if __name__ == "__main__":
    parser = transformers.HfArgumentParser((ModelArguments, DataArguments, EvalArguments))
    model_args, data_args, eval_args = parser.parse_args_into_dataclasses()

    logging.basicConfig(level=logging.INFO)
    main(data_args, model_args, eval_args)


tigerscore/distill/test_llama_sum.py
TIGER-AI-Lab/TIGERScore
import json
import transformers
import torch
import logging
import sys
import regex as re
from pathlib import Path
from tqdm import tqdm
from dataclasses import dataclass, field
from typing import Optional, Sequence, Dict, List
from torch.utils.data import Dataset, DataLoader
from string import Template
from mt_metrics_eval.stats import Correlation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.process_utils import get_xgptscore_from_json, json_postprocess
from peft import PeftModel
from vllm import LLM, SamplingParams
import argparse


MAX_INT = sys.maxsize

IGNORE_INDEX = -100
DEFAULT_PAD_TOKEN = "[PAD]"
DEFAULT_EOS_TOKEN = "</s>"
DEFAULT_BOS_TOKEN = "<s>"
DEFAULT_UNK_TOKEN = "<unk>"
PROMPT_DICT = {
    "prompt_input": (
        "Below is an instruction that describes a task, paired with an input that provides further context. "
        "Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:"
    ),
    "prompt_no_input": (
        "Below is an instruction that describes a task. "
        "Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Response:"
    ),
}
FINETUNE_INST = "You are evaluating errors in a model-generated output for a(an) ${task} task."
FINETUNE_INPUT = """\
Task instruction: ${generation_instruction}
Source: ${input_context}
Model-generated Output: ${hypothesis_output}

Based on the given task instruction and source, identify errors in this model-generated output.
For each error you give in the response, please also elaborate the following information:
- error location (the words that are wrong in the output)
- error aspect it belongs to.
- explanation why it's an error, and the correction suggestions.
- severity of the error ("Major" or "Minor"). 
- reduction of score (an interger between 1 and 5 given the severity of the error)

Your evaluation output:
"""

def get_sum_penalties(eval_output: dict):
    """
    Args:
        eval_output: dict, the json output of the eval function
    
    Returns:
    """
    try:
        penalty_score = 0
        for aspect in eval_output:
            for penalty_point in eval_output[aspect]["penalty_points"]:
                penalty_score += penalty_point["score_reduction"]
        return - penalty_score
    except:
        return None

def get_torch_dtype(dtype_str):
    """
        Get the torch dtype from a string
    """
    if dtype_str == "float32":
        return torch.float32
    elif dtype_str == "float16":
        return torch.float16
    elif dtype_str == "bfloat16":
        return torch.bfloat16
    elif dtype_str == "int8":
        return torch.int8
    else:
        raise ValueError("Invalid dtype {}".format(dtype_str))
    
    
def batch_data(data_list, batch_size=1):
    n = len(data_list) // batch_size
    batch_data = []
    for i in range(n-1):
        start = i * batch_size
        end = (i+1)*batch_size
        batch_data.append(data_list[start:end])

    last_start = (n-1) * batch_size
    last_end = MAX_INT
    batch_data.append(data_list[last_start:last_end])
    return batch_data


class MyCorrelation(Correlation):
    def __init__(self, num_sys:int, gold_scores:List[int], metric_scores:List[int]):
        # remove nan in metrics scores
        none_metric_scores_idxs = [idx for idx, x in enumerate(metric_scores) if x is None]
        logging.info("Remove {} nan scores from {} scores".format(
            len(none_metric_scores_idxs),
            len(metric_scores)
        ))
        gold_scores = gold_scores.copy()
        # set gold scores to None if metric scores are None
        for idx in none_metric_scores_idxs[::-1]:
            gold_scores[idx] = None 
        super().__init__(num_sys, gold_scores, metric_scores)
        
def main(args):

    if args.output_path is not None:
        output_file = Path(args.output_path)
    else:
        output_file = Path(args.data_path).with_suffix('.xgptscore.output.json')
    if not output_file.exists() or args.overwrite:
        logging.info("Loading model...")
        sampling_params = SamplingParams(temperature=0, top_p = 1, max_tokens=1024)
        llm = LLM(model=args.model_name_or_path,tensor_parallel_size=1)
        logging.info("Model loaded from {}".format(args.model_name_or_path))
    
        eval_outputs = []
        
        
        logging.info("Load input data from {}".format(args.data_path))
        with open(args.data_path, "r") as f:
            input_data = json.load(f)
        formatted_data = []
        for item in input_data:
            for cand in item['candidates']:
                inst = Template(FINETUNE_INST).substitute(task=args.task)
                input_ = Template(FINETUNE_INPUT).substitute(
                    task=args.task,
                    generation_instruction=item['instruction'],
                    input_context=item['input'],
                    hypothesis_output=cand['text'],
                )
                formatted_data.append({
                    "instruction": inst,
                    "input": input_,
                })
            prompt_sources = [example['instruction'] + '\n' + example['input'] for example in formatted_data]
            prompt_sources = [x.strip(' \n') + "\n" for x in prompt_sources]
        
        batch_prompts = batch_data(prompt_sources, batch_size=args.batch_size)
        
        for idx,batch_prompt in enumerate(batch_prompts):
            if isinstance(batch_prompt, list):
                pass
            else:
                batch_prompt = [batch_prompt]
            
            completions = llm.generate(batch_prompt, sampling_params)
            for output in completions:
                generated_text = output.outputs[0].text
                eval_outputs.append(generated_text)
        
        cand_idx = 0
        for idx,(item, eval_output) in enumerate(zip(input_data, eval_outputs)):
            for cand in item['candidates']:
                cand['eval_output'] = eval_outputs[cand_idx]
                score_reductions = re.findall(r"(?<=\nScore reduction \d+: )(\d+\.\d+|\d+)", eval_outputs[cand_idx])
                cand['xgptscore'] = -sum(map(float, score_reductions))
                cand_idx += 1
        
        with open(output_file, 'w') as f:
            json.dump(input_data, f, indent=4, ensure_ascii=False)
        logging.info("Saved eval results to {}".format(output_file))
    else:
        with open(output_file, 'r') as f:
            input_data = json.load(f)
        for ex in input_data:
            for cand in ex['candidates']:
                score_reductions = re.findall(r"(?<=\nScore reduction \d+: )(\d+\.\d+|\d+)", cand['eval_output'])
                cand['xgptscore'] = -sum(map(float, score_reductions))
        with open(output_file, 'w') as f:
            json.dump(input_data, f, indent=4, ensure_ascii=False)
        logging.info("Loaded eval results from {}".format(output_file))
    # Compute correlation
    human_score_names = args.human_score_names.split(',')
    
    
    for h_name in human_score_names:
        human_scores = []
        xgptscores = []
        for item in input_data:
            for cand in item['candidates']:
                for s_name, score in cand['scores'].items():
                        if s_name == h_name:
                            xgptscores.append(cand['xgptscore'])
                            human_scores.append(score)
                            break
        corr = MyCorrelation(1, human_scores, xgptscores)
        logging.info("Human score: {}".format(h_name))
        logging.info("Pearson correlation: {}".format(corr.Pearson()))
        logging.info("Spearman correlation: {}".format(corr.Spearman()))
        logging.info("Kendall correlation: {}".format(corr.Kendall()))


if __name__ == "__main__":

    logging.basicConfig(level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument("--model_name_or_path", type=str, default=None)
    parser.add_argument("--data_path", type=str, default=None)
    parser.add_argument("--output_path", type=str, default=None)
    parser.add_argument("--overwrite", action="store_true")
    parser.add_argument("--task", type=str, default="summarization")
    parser.add_argument("--batch_size", type=int, default=1)
    parser.add_argument("--human_score_names", type=str, default="score")
    args = parser.parse_args()
    main(args)


tigerscore/distill/test_llama_vllm.py
TIGER-AI-Lab/TIGERScore
#    Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import copy
import logging
from dataclasses import dataclass, field
from typing import Dict, Optional, Sequence
from pathlib import Path


import torch
import transformers
import utils
from tqdm import tqdm
from torch.utils.data import Dataset
from trainer import Trainer, CustomLoraTrainer
from peft import get_peft_config, get_peft_model, LoraConfig, TaskType, prepare_model_for_int8_training

IGNORE_INDEX = -100
DEFAULT_PAD_TOKEN = "[PAD]"
DEFAULT_EOS_TOKEN = "</s>"
DEFAULT_BOS_TOKEN = "<s>"
DEFAULT_UNK_TOKEN = "<unk>"
PROMPT_DICT = {
    "prompt_input": (
        "Below is an instruction that describes a task, paired with an input that provides further context. "
        "Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:"
    ),
    "prompt_no_input": (
        "Below is an instruction that describes a task. "
        "Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Response:"
    ),
}


@dataclass
class ModelArguments:
    model_name_or_path: Optional[str] = field(default="facebook/opt-125m")
    is_lora: bool = field(default=False)


@dataclass
class DataArguments:
    train_data_path: str = field(default=None, metadata={"help": "Path to the training data."})
    eval_data_path: str = field(default=None, metadata={"help": "Path to the evaluation data."})

@dataclass
class TrainingArguments(transformers.TrainingArguments):
    cache_dir: Optional[str] = field(default=None)
    optim: str = field(default="adamw_torch")
    model_max_length: int = field(
        default=512,
        metadata={"help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)."},
    )


def smart_tokenizer_and_embedding_resize(
    special_tokens_dict: Dict,
    tokenizer: transformers.PreTrainedTokenizer,
    model: transformers.PreTrainedModel,
):
    """Resize tokenizer and embedding.

    Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
    """
    num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)
    model.resize_token_embeddings(len(tokenizer))

    if num_new_tokens > 0:
        input_embeddings = model.get_input_embeddings().weight.data
        output_embeddings = model.get_output_embeddings().weight.data

        input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)
        output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)

        input_embeddings[-num_new_tokens:] = input_embeddings_avg
        output_embeddings[-num_new_tokens:] = output_embeddings_avg


def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer) -> Dict:
    """Tokenize a list of strings."""
    tokenized_list = [
        tokenizer(
            text,
            return_tensors="pt",
            padding="longest",
            max_length=tokenizer.model_max_length,
            truncation=True,
        )
        for text in tqdm(strings, desc="Tokenizing...")
    ]
    input_ids = labels = [tokenized.input_ids[0] for tokenized in tokenized_list]
    input_ids_lens = labels_lens = [
        tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item() for tokenized in tokenized_list
    ]
    return dict(
        input_ids=input_ids,
        labels=labels,
        input_ids_lens=input_ids_lens,
        labels_lens=labels_lens,
    )


def preprocess(
    sources: Sequence[str],
    targets: Sequence[str],
    tokenizer: transformers.PreTrainedTokenizer,
) -> Dict:
    """Preprocess the data by tokenizing."""
    examples = [s + t for s, t in zip(sources, targets)]
    examples_tokenized, sources_tokenized = [_tokenize_fn(strings, tokenizer) for strings in (examples, sources)]
    input_ids = examples_tokenized["input_ids"]
    labels = copy.deepcopy(input_ids)
    for label, source_len in zip(labels, sources_tokenized["input_ids_lens"]):
        label[:source_len] = IGNORE_INDEX
    return dict(input_ids=input_ids, labels=labels)


class SupervisedDataset(Dataset):
    """Dataset for supervised fine-tuning."""

    def __init__(self, data_path: str, tokenizer: transformers.PreTrainedTokenizer):
        super(SupervisedDataset, self).__init__()
        logging.warning("Loading data...")
        list_data_dict = utils.jload(data_path)

        logging.warning("Formatting inputs...")
        prompt_input, prompt_no_input = PROMPT_DICT["prompt_input"], PROMPT_DICT["prompt_no_input"]
        # sources = [
        #     prompt_input.format_map(example) if example.get("input", "") != "" else prompt_no_input.format_map(example)
        #     for example in list_data_dict
        # ]
        sources = [example['instruction'] + "\n" + example['input'] for example in list_data_dict]
        sources = [x.strip(' \n') + "\n" for x in sources]
        targets = [example['output'] + tokenizer.eos_token for example in list_data_dict]

        logging.warning("Tokenizing inputs... This may take some time...")
        data_dict = preprocess(sources, targets, tokenizer)
        self.input_ids = data_dict["input_ids"]
        self.labels = data_dict["labels"]

    def __len__(self):
        return len(self.input_ids)

    def __getitem__(self, i) -> Dict[str, torch.Tensor]:
        return dict(input_ids=self.input_ids[i], labels=self.labels[i])


@dataclass
class DataCollatorForSupervisedDataset(object):
    """Collate examples for supervised fine-tuning."""

    tokenizer: transformers.PreTrainedTokenizer

    def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
        input_ids, labels = tuple([instance[key] for instance in instances] for key in ("input_ids", "labels"))
        input_ids = torch.nn.utils.rnn.pad_sequence(
            input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id
        )
        labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=IGNORE_INDEX)
        
        return dict(
            input_ids=input_ids,
            labels=labels,
            attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
        )


def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer, data_args) -> Dict:
    """Make dataset and collator for supervised fine-tuning."""
    train_dataset = SupervisedDataset(tokenizer=tokenizer, data_path=data_args.train_data_path)
    data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)
    if data_args.eval_data_path:
        eval_dataset = SupervisedDataset(tokenizer=tokenizer, data_path=data_args.eval_data_path)
    else:
        eval_dataset = None
    return dict(train_dataset=train_dataset, eval_dataset=eval_dataset, data_collator=data_collator)



def train():
    parser = transformers.HfArgumentParser((ModelArguments, DataArguments, TrainingArguments))
    model_args, data_args, training_args = parser.parse_args_into_dataclasses()

    if model_args.is_lora:
        logging.warning("Using LoRA model...")
        model = transformers.AutoModelForCausalLM.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=training_args.cache_dir,
            device_map="auto",
            load_in_8bit=True,
            torch_dtype=torch.bfloat16,
            use_auth_token=True,
        )
        # target_modules = ["q_proj", "k_proj", "v_proj", "out_proj", "fc_in", "fc_out", "wte"]
        target_modules = ["q_proj", "k_proj"]
        peft_config = LoraConfig(
            task_type=TaskType.CAUSAL_LM, inference_mode=False, 
            r=8, lora_alpha=16, lora_dropout=0.05,
            target_modules=target_modules,
        )
        logging.warning(f"Using LoRA config: {peft_config}")
        model = prepare_model_for_int8_training(model)
        model = get_peft_model(model, peft_config)
        model.print_trainable_parameters()
    else:
        model = transformers.AutoModelForCausalLM.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=training_args.cache_dir,
            use_auth_token=True,
            torch_dtype=torch.bfloat16,
        )


    tokenizer = transformers.AutoTokenizer.from_pretrained(
        model_args.model_name_or_path,
        cache_dir=training_args.cache_dir,
        model_max_length=training_args.model_max_length,
        padding_side="right",
        use_fast=False,
        use_auth_token=True,
    )
    special_tokens_dict = dict()
    if tokenizer.pad_token is None:
        special_tokens_dict["pad_token"] = DEFAULT_PAD_TOKEN
    if tokenizer.eos_token is None:
        special_tokens_dict["eos_token"] = DEFAULT_EOS_TOKEN
    if tokenizer.bos_token is None:
        special_tokens_dict["bos_token"] = DEFAULT_BOS_TOKEN
    if tokenizer.unk_token is None:
        special_tokens_dict["unk_token"] = DEFAULT_UNK_TOKEN

    smart_tokenizer_and_embedding_resize(
        special_tokens_dict=special_tokens_dict,
        tokenizer=tokenizer,
        model=model,
    )

    data_module = make_supervised_data_module(tokenizer=tokenizer, data_args=data_args)
    if model_args.is_lora:
        logging.warning("Using LoRA trainer...")
        trainer = CustomLoraTrainer(model=model, tokenizer=tokenizer, args=training_args, **data_module)
    else:
        logging.warning("Using default trainer...")
        trainer = Trainer(model=model, tokenizer=tokenizer, args=training_args, **data_module)
    trainer.train(
        resume_from_checkpoint=training_args.resume_from_checkpoint,
    )
    trainer.save_state()
    best_checkpoint_dir = Path(training_args.output_dir) / "checkpoint-best"
    trainer.save_model(output_dir=best_checkpoint_dir)
    # trainer.push_to_hub(training_args.run_name)

if __name__ == "__main__":
    train()
tigerscore/distill/train.py
TIGER-AI-Lab/TIGERScore
from typing import Optional
import os
from transformers.trainer import *
from peft import PeftModel
class CustomLoraTrainer(Trainer):
    def _save(self, output_dir: Optional[str] = None, state_dict=None):
        # If we are executing this function, we are the process zero, so we don't check for that.
        output_dir = output_dir if output_dir is not None else self.args.output_dir
        os.makedirs(output_dir, exist_ok=True)
        logger.info(f"Saving model checkpoint to {output_dir}")
        # Save a trained model and configuration using `save_pretrained()`.
        # They can then be reloaded using `from_pretrained()`
        if not isinstance(self.model, PreTrainedModel) and not isinstance(self.model, PeftModel):
            if state_dict is None:
                state_dict = self.model.state_dict()

            if isinstance(unwrap_model(self.model), PreTrainedModel):
                unwrap_model(self.model).save_pretrained(
                    output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors
                )
            else:
                logger.info("Trainer.model is not a `PreTrainedModel`, only saving its state dict.")
                if self.args.save_safetensors:
                    safetensors.torch.save_file(state_dict, os.path.join(output_dir, SAFE_WEIGHTS_NAME))
                else:
                    torch.save(state_dict, os.path.join(output_dir, WEIGHTS_NAME))
        else:
            print("Saving LoRA model...")
            self.model.save_pretrained(
                output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors
            )

        if self.tokenizer is not None:
            self.tokenizer.save_pretrained(output_dir)

        # Good practice: save your training arguments together with the trained model
        torch.save(self.args, os.path.join(output_dir, TRAINING_ARGS_NAME))
tigerscore/distill/trainer.py
TIGER-AI-Lab/TIGERScore
import dataclasses
import logging
import math
import os
import io
import sys
import time
import json
from typing import Optional, Sequence, Union, Dict

import openai
import tqdm
from openai import openai_object
import copy

StrOrOpenAIObject = Union[str, openai_object.OpenAIObject]

openai_org = os.getenv("OPENAI_ORG")
if openai_org is not None:
    openai.organization = openai_org
    logging.warning(f"Switching to organization: {openai_org} for OAI API key.")


@dataclasses.dataclass
class OpenAIDecodingArguments(object):
    max_tokens: int = 1800
    temperature: float = 0.2
    top_p: float = 1.0
    n: int = 1
    stream: bool = False
    stop: Optional[Sequence[str]] = None
    presence_penalty: float = 0.0
    frequency_penalty: float = 0.0
    suffix: Optional[str] = None
    logprobs: Optional[int] = None
    echo: bool = False


def openai_completion(
    prompts: Union[str, Sequence[str], Sequence[Dict[str, str]], Dict[str, str]],
    decoding_args: OpenAIDecodingArguments,
    model_name="text-davinci-003",
    sleep_time=2,
    batch_size=1,
    max_instances=sys.maxsize,
    max_batches=sys.maxsize,
    return_text=False,
    **decoding_kwargs,
) -> Union[Union[StrOrOpenAIObject], Sequence[StrOrOpenAIObject], Sequence[Sequence[StrOrOpenAIObject]],]:
    """Decode with OpenAI API.

    Args:
        prompts: A string or a list of strings to complete. If it is a chat model the strings should be formatted
            as explained here: https://github.com/openai/openai-python/blob/main/chatml.md. If it is a chat model
            it can also be a dictionary (or list thereof) as explained here:
            https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb
        decoding_args: Decoding arguments.
        model_name: Model name. Can be either in the format of "org/model" or just "model".
        sleep_time: Time to sleep once the rate-limit is hit.
        batch_size: Number of prompts to send in a single request. Only for non chat model.
        max_instances: Maximum number of prompts to decode.
        max_batches: Maximum number of batches to decode. This argument will be deprecated in the future.
        return_text: If True, return text instead of full completion object (which contains things like logprob).
        decoding_kwargs: Additional decoding arguments. Pass in `best_of` and `logit_bias` if you need them.

    Returns:
        A completion or a list of completions.
        Depending on return_text, return_openai_object, and decoding_args.n, the completion type can be one of
            - a string (if return_text is True)
            - an openai_object.OpenAIObject object (if return_text is False)
            - a list of objects of the above types (if decoding_args.n > 1)
    """
    is_single_prompt = isinstance(prompts, (str, dict))
    if is_single_prompt:
        prompts = [prompts]

    if max_batches < sys.maxsize:
        logging.warning(
            "`max_batches` will be deprecated in the future, please use `max_instances` instead."
            "Setting `max_instances` to `max_batches * batch_size` for now."
        )
        max_instances = max_batches * batch_size

    prompts = prompts[:max_instances]
    num_prompts = len(prompts)
    prompt_batches = [
        prompts[batch_id * batch_size : (batch_id + 1) * batch_size]
        for batch_id in range(int(math.ceil(num_prompts / batch_size)))
    ]

    completions = []
    for batch_id, prompt_batch in tqdm.tqdm(
        enumerate(prompt_batches),
        desc="prompt_batches",
        total=len(prompt_batches),
    ):
        batch_decoding_args = copy.deepcopy(decoding_args)  # cloning the decoding_args

        while True:
            try:
                shared_kwargs = dict(
                    model=model_name,
                    **batch_decoding_args.__dict__,
                    **decoding_kwargs,
                )
                completion_batch = openai.Completion.create(prompt=prompt_batch, **shared_kwargs)
                choices = completion_batch.choices

                for choice in choices:
                    choice["total_tokens"] = completion_batch.usage.total_tokens
                completions.extend(choices)
                break
            except openai.error.OpenAIError as e:
                logging.warning(f"OpenAIError: {e}.")
                if "Please reduce your prompt" in str(e):
                    batch_decoding_args.max_tokens = int(batch_decoding_args.max_tokens * 0.8)
                    logging.warning(f"Reducing target length to {batch_decoding_args.max_tokens}, Retrying...")
                else:
                    logging.warning("Hit request rate limit; retrying...")
                    time.sleep(sleep_time)  # Annoying rate limit on requests.

    if return_text:
        completions = [completion.text for completion in completions]
    if decoding_args.n > 1:
        # make completions a nested list, where each entry is a consecutive decoding_args.n of original entries.
        completions = [completions[i : i + decoding_args.n] for i in range(0, len(completions), decoding_args.n)]
    if is_single_prompt:
        # Return non-tuple if only 1 input and 1 generation.
        (completions,) = completions
    return completions


def _make_w_io_base(f, mode: str):
    if not isinstance(f, io.IOBase):
        f_dirname = os.path.dirname(f)
        if f_dirname != "":
            os.makedirs(f_dirname, exist_ok=True)
        f = open(f, mode=mode)
    return f


def _make_r_io_base(f, mode: str):
    if not isinstance(f, io.IOBase):
        f = open(f, mode=mode)
    return f


def jdump(obj, f, mode="w", indent=4, default=str):
    """Dump a str or dictionary to a file in json format.

    Args:
        obj: An object to be written.
        f: A string path to the location on disk.
        mode: Mode for opening the file.
        indent: Indent for storing json dictionaries.
        default: A function to handle non-serializable entries; defaults to `str`.
    """
    f = _make_w_io_base(f, mode)
    if isinstance(obj, (dict, list)):
        json.dump(obj, f, indent=indent, default=default)
    elif isinstance(obj, str):
        f.write(obj)
    else:
        raise ValueError(f"Unexpected type: {type(obj)}")
    f.close()


def jload(f, mode="r"):
    """Load a .json file into a dictionary."""
    f = _make_r_io_base(f, mode)
    jdict = json.load(f)
    f.close()
    return jdict

tigerscore/distill/utils.py
TIGER-AI-Lab/TIGERScore
import os
import sys
import json
import argparse
import pickle
from pathlib import Path

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, required=True)
    parser.add_argument('--task', type=str, required=True)
    parser.add_argument('--rm_old', action='store_true')
    
    args = parser.parse_args()
    data_dir = args.data_dir

    task_dir = Path(data_dir) / args.task
    for data_file in os.listdir(task_dir):
        if not data_file.endswith('.pkl'):
            continue
        print("Data file: ", data_file)
        data_path = task_dir / data_file
        with open(data_path, 'rb') as f:
            data = pickle.load(f)
        print("# of data: ", len(data))
        if isinstance(data, dict):
            print("Data Example: ", data[list(data.keys())[0]])
        elif isinstance(data, list):
            print("Data example: ", data[0])
        with open(data_path.with_suffix('.json'), 'w') as f:
            json.dump(data, f, indent=4)
        if args.rm_old:
            data_path.unlink()

tigerscore/download_dataset/bartscore_data_process.py
TIGER-AI-Lab/TIGERScore
"""
    This script downloads the dataset and splits it into train, val, test
    This file is modified based on:
    https://github.com/Ravoxsg/SummaReranker-ACL-22-/blob/main/src/candidate_generation/main_download_dataset.py
    We thank the authors for sharing their code.
"""

import os
import argparse
import sys
import datasets
import json
import PIL
import numpy as np
from tqdm import tqdm
from pathlib import Path
from typing import List, Tuple, Union
from datasets import load_dataset_builder
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from common.utils import (
    generate_hash_code,
    str2bool,
    empty2None,
    fetch_single_image
)
from common.datasets_config import DATASETS_CONFIG
from pathlib import Path
from typing import List, Tuple, Union

def preprocess_example(
    ex: dict,
    task: str,
    dataset: str,
    dataset_version: str,
    input_key: str,
    output_key: str,
    id_key: str=None,
    dataset_cache_dir: str=None,
):
    """
    This function preprocesses the batch data according to the task and datasets
    Args:
        ex: a batch of data
        task: the task of the dataset
        dataset: the name of the dataset
        dataset_version: the version of the dataset
    Returns:
        id: the id of the example
        inst: the instruction of the example
        input: the input of the example
        output: the output of the example
    """
    if dataset_cache_dir is not None:
        dataset_cache_dir = Path(dataset_cache_dir)
    else:
        dataset_cache_dir = Path.cwd() / "../../data" / dataset / dataset_version / "cache"
    inst = None
    # task and dataset specific processing
    if task == "translation":
        input = ex["translation"][input_key]
        output = ex["translation"][output_key]
    elif task == "data2text":
        if "dart" in dataset:
            input = tub2str_dart(ex[input_key])
            output = ex[output_key]["text"][-1]
        elif "totto" in dataset: 
            from preprocess_utils_totto import get_highlighted_subtable
            subtable = get_highlighted_subtable(table=ex['table'], cell_indices=ex['highlighted_cells'], with_heuristic_headers=True)
            input = process_totto(subtable,ex)
            output = ex[output_key]["final_sentence"][-1]
        elif "logicnlg" in dataset:
            input = process_logicnlg(ex)
            output = ex[output_key]                   
        elif "wikitabletext" in dataset:
            input = process_wikitabletext(ex)
            output = ex[output_key]
        elif "web_nlg" in dataset:
            input = "\n".join([f"({s})" for s in ex[input_key]["mtriple_set"][0]]).strip()
            
            output = ex[output_key]["text"][-1]
        else:
            input = ex[input_key]
            output = ex[output_key]
    elif task == "image_captioning":
        dataset_cache_dir.mkdir(parents=True, exist_ok=True)
        input = ex[input_key]
        output = ex[output_key]
        if isinstance(input, str):
            # input is a url
            # filter out urls that are invalid
            try:
                image = fetch_single_image(input, retries=0, timeout=3)
            except Exception as e:
                return None, None, None
        elif isinstance(input, PIL.Image.Image):
            # input is PIL image
            image = input.convert("RGB")
            # save image 
            image_path = Path(dataset_cache_dir) / f"{generate_hash_code(output)}.jpg"
            image.save(image_path)
            input = image_path.absolute().as_posix()
        else:
            raise ValueError("Invalid input type: {}".format(type(input)))
    elif task == "long-form QA":
        if "asqa" in dataset:
            input = ex['ambiguous_question']
            knowledges = [x['content'] for x in ex['annotations'][0]["knowledge"] if x['content'] is not None]
            output = ex['annotations'][0]['long_answer']
        elif "natural_questions" in dataset:
            input = ex[input_key]
            output = ex[output_key]
            if len(output) <= 100:
                output = None # skip this example, we want long answer
        elif "cosmos_qa" in dataset:
            input = "Context: " + ex['context'] + "\n" + "Question: " + ex['question'] + "\n" + "Answer: "
            output = ex[f"answer{ex['label']}"]
            if "None of the above choices ." in output or len(output) <= 50:
                output = None # skip this example, we want long answer
        elif "eli5" in dataset:
            input = ex[input_key]
            if len(ex[output_key]) > 0:
                highest_score_idx = np.argmax(ex[output_key]['score'])
                answer = ex['answers']['text'][highest_score_idx]
                output = answer
            else:
                output = None
        elif "FeTaQA" in dataset:
            input = process_fetaqa(ex)
            output = ex[output_key]
            if len(output) <= 100:
                output = None
            pass
        else:
            input = ex[input_key]
            output = ex[output_key]
    elif task == "story_generation":
        if "roc" in dataset:
            input = "Title: " + ex['storytitle'] + "\n" + "Story: \n"
            for i in range(1, 5):
                input += "Sentence " + str(i) + ": " + ex['sentence' + str(i)] + "\n"
            input += "Sentence 5 (Ending): "
            output = ex['sentence5']
        elif "hellaswag" in dataset:
            input = "Context" + ex['ctx']
            output = ex['endings'][int(ex['label'])]
        elif dataset == "swag":
            input = ex['startphrase']
            output = ex['ending{}'.format(ex['label'])]
        else:
            input = ex[input_key]
            output = ex[output_key]
    elif task == "other":
        if "common_gen" in dataset:
            input = "Concepts: " + ", ".join(ex['concepts'])
            output = ex['target']
        elif "alpaca-gpt4" in dataset:
            input = ex["instruction"] + "\n" + ex["input"]
            output = ex[output_key]
        else:
            input = ex[input_key]
            output = ex[output_key]
    elif task == "mathQA":
        if "math_qa" in dataset:
            input = ex[input_key] #+ "\n Options: " + ex['options']
            output = ex[output_key]
        else:
            input = ex[input_key]
            output = ex[output_key]
        # input += "Let's think step by step.\n" # We can move this into instruction instead of input
    elif task == "code":
        if "code_contests" in dataset:
            input = ex[input_key]
            solution = ex[output_key]["solution"]
            solution = sorted(solution, key=lambda x:len(x))
            output = solution[0] if len(solution) > 0 else None
        else:
            input = ex[input_key]
            output = ex[output_key]
    elif task == "instruction-following":
        if "lima" in dataset:
            if len(ex['conversations']) != 2:
                input = None
                output = None
            else:
                input = ex['conversations'][0]
                output = ex['conversations'][1]
        elif "alpaca" in dataset or "Guanaco" in dataset:
            inst = ex["instruction"]
            input = ex["input"]
            output = ex["output"]
        elif "oasst1_en" in dataset:
            if len(ex['messages']) != 2:
                input = None
                output = None
            else:
                input = ex['messages'][0]['content']
                output = ex['messages'][1]['content']
        elif "dolly" in dataset:
            inst = ex["instruction"]
            input = ex["context"]
            output = ex["response"]
    else:
        input = ex[input_key]
        output = ex[output_key]

    # print("input: ", input)
    # print("output: ", output)
    if id_key is not None:
        id = ex[id_key]
    elif input is not None and output is not None:
        id = generate_hash_code(input + output)
    else:
        id = None

    return id, inst, input, output

def download_dataset(
    task: str,
    dataset: str,
    dataset_version: str,
    split: str,
    input_key: str,
    output_key: str,
    set_name: str,
    instruction: str=None,
    save_dir: Union[str, Path]=None,
    use_auth_token: dict = None,
    streaming: bool = False,
    shuffle: bool = False,
    seed: int = 42,
    overwrite: bool = False,
    id_key: str = None,
) -> None:
    data_dir = save_dir
    if data_dir is None:
        data_dir = Path(os.path.dirname(__file__)).parent.parent / "data" 
    dataset_dir = data_dir / dataset / (dataset_version or "")
    dataset_dir.mkdir(parents=True, exist_ok=True)
    save_file = dataset_dir / f"{set_name}_data.json"
    print("Downloading {}:{}:{}".format(dataset, dataset_version, split))
    print("To save at: {}".format(save_file))
    if Path(save_file).exists():
        print("File already exists.")
        if not overwrite:
            print("Skipping download...")
            return
        else:
            print("Overwriting file at: {}".format(save_file))

    # check split
    if len(split.split("+")) > 1:
        splits = split.split("+")
    else:
        splits = [split]
    examples = []

    for _split in splits:
        start_idx = _split[_split.find("[")+1:_split.find(":")]
        end_idx = _split[_split.find(":")+1:_split.find("]")]
        __split = _split[:_split.find("[")]
                                  
        if start_idx == "":
            start_idx = 0
        else:
            start_idx = int(start_idx)
        if end_idx == "":
            end_idx = None
            num_examples = None
        else:
            end_idx = int(end_idx)
            num_examples = end_idx - start_idx
            if num_examples < 0:
                raise ValueError("0 examples selected. Please check your split: {}".format(_split))
        
        # load dataset
        try:
            # Try Streaming
            _streaming = streaming
            DS = datasets.load_dataset(
                dataset, dataset_version,
                split=__split, streaming=_streaming,
                use_auth_token=use_auth_token)
            _exs = [x for x in DS.take(100)]
        except Exception as e:
            print(e)
            print("Error loading dataset. Trying again without streaming...")
            _streaming = False
            DS = datasets.load_dataset(
                dataset, dataset_version,
                split=__split, streaming=_streaming, verification_mode='no_checks',
                use_auth_token=use_auth_token)
        print("Dataset loaded.")
        if shuffle:
            DS = DS.shuffle(seed=seed)
        
        if _streaming:
            iter_DS = DS.skip(start_idx)
        else:
            if start_idx > 0:
                iter_DS = [ex for ex in DS][start_idx:]
            else:
                iter_DS = DS
        # Process the examples
        unique_ids = set()
        with tqdm(desc="Processing {}:{}:{}".format(dataset, dataset_version, _split), 
            total=num_examples,
        ) as pbar:
            for i, ex in enumerate(iter_DS):
                # task and dataset specific processing
                dataset_cache_dir = dataset_dir / "cache"
                _id, _inst, _input, _output = preprocess_example(
                    ex, task, dataset, dataset_version, input_key, output_key, id_key, str(dataset_cache_dir))

               
                if _input is None or _output is None:
                    continue
                _datasource = dataset
                _datasource += f":{dataset_version}" if dataset_version else ""
                               
                if _id not in unique_ids:
                    examples.append({
                        "id": _id,
                        "instruction": _inst or instruction,
                        "input": _input, # input after applying the template
                        "output": _output,
                        "data_source": _datasource,
                        "task": task,
                    })
                    unique_ids.add(_id)
                    pbar.update(1)
                if num_examples is not None and len(unique_ids) >= num_examples:
                    break
    
    # Save the dataset
    with open(save_file, "w") as f:
        json.dump(examples, f, indent=4, ensure_ascii=False)    
    print(f"Saved {len(examples)} examples to {save_file}")
    
def tub2str_dart(tub: List[List[str]]):
    tub_str = ""
    for row in tub:
        tub_str += "( "
        for col in row[:-1]:
            tub_str += col + ", "
        tub_str += row[-1]
        tub_str += ")\n"
    return tub_str #+ '\n Start describing : '

def process_logicnlg(ex:dict):
    d = eval(ex['table'])
    tmp = ''
    
    for i in range(1,len(d)):
        tmp += 'In row {} , '.format(i)
        for j, s in enumerate(d[i]):
            entity = str(s)
            tmp += 'the {} is {} , '.format(d[0][j], entity)
        tmp = tmp[:-3] + ' . '

    tmp_prefix = 'Given the table title of "{}" . '.format(ex['title'])
    return tmp_prefix + tmp #+ '\n Start describing : '

def process_wikitabletext(ex:dict):
    d = eval(ex['content'])
    header = eval(ex['headers'])
    tmp = ''
    
    for i,s in enumerate(header):
        tmp += 'the {} is {} , '.format(s, d[i])
    tmp = tmp[:-3] + ' . '
    return tmp #+ '\n Start describing : '

def process_totto(subtable:list,ex:dict):
    table_str = ""
    
    if ex['table_page_title']:
        table_str += "Given the table title of {} . ".format(ex['table_page_title'])
    if ex['table_section_title']:
        table_str += "Given the table section title of {} . ".format(ex['table_section_title'])


    for item in subtable:
        cell = item["cell"]
        row_headers = item["row_headers"]
        col_headers = item["col_headers"]

        # The value of the cell.
        item_str = "Given the cell value of {} . ".format(cell["value"])

        # All the column headers associated with this cell.
        if col_headers:
            item_str += "The column header : "           
            col_headers_set = set()
            
            for col_header in col_headers:
                col_headers_set.add(col_header["value"])
            
            for col_header in col_headers_set:
              item_str += " {} , ".format(col_header)

            item_str = item_str[:-3] + " . "
        if row_headers:
            item_str += "The row header : "
            row_headers_set = set()
            
            for row_header in row_headers:
                row_headers_set.add(row_header["value"])
            for row_header in row_headers_set:
                item_str += " {} , ".format(row_header)
                
            item_str = item_str[:-3] + " . "
        
        
        # All the row headers associated with this cell.
        for row_header in row_headers:
          item_str += "The row header is {} . ".format(row_header["value"])
        table_str += item_str

    return table_str #+ '\n Start describing : '

def process_fetaqa(ex:dict):
    d = ex['table_array']
    highlights = ex['highlighted_cell_ids']
    highlights.sort()
    tmp = ''
    old_row = -1
    
    for row,col in highlights:
        if row != old_row:
            tmp = tmp[:-3] + ' . ' 
            tmp += 'In row {} , '.format(row)
            old_row = row

        tmp += 'the {} is {} , '.format(d[0][col], d[row][col])
        
    tmp = tmp[3:]
    tmp = tmp[:-3] + ' . '

    tmp_prefix = 'Given the table title of "{}" . '.format(ex['table_section_title'])
    question = ex['question']
    return tmp_prefix + tmp + question


def main(args):
    data_dir = args.data_dir
    if data_dir is None:
        data_dir = Path(os.path.dirname(__file__)).parent.parent / "data"
    data_dir.mkdir(parents=True, exist_ok=True)
    datasets_config = DATASETS_CONFIG[args.task]
    for key in datasets_config.keys():
        if ':' not in key:
            dataset, dataset_version = key, None
        else:
            dataset, dataset_version = key.split(":")
        print("Loading dataset: {}:{}...".format(dataset, dataset_version))
        print("Dataset Details:")
        print("Dataset Name: {}".format(dataset))
        print("Dataset Version: {}".format(dataset_version))
        split_info = datasets_config[key]['split_info']
        instruction = datasets_config[key]['instruction']
        input_key = datasets_config[key]['input_key']
        output_key = datasets_config[key]['output_key']

        for set_name, split in split_info.items():
            download_dataset(
                task=args.task,
                dataset=dataset,
                dataset_version=dataset_version,
                split=split,
                instruction=instruction,
                input_key=input_key,
                output_key=output_key,
                set_name=set_name,
                save_dir=data_dir,
                use_auth_token=args.hf_use_auth_token,
                streaming=args.streaming,
                shuffle=args.shuffle,
                seed=args.seed,
                overwrite=args.overwrite,
            )

if __name__ == "__main__":


    parser = argparse.ArgumentParser()

    parser.add_argument('--seed', type = int, default = 42)
    # data
    parser.add_argument('--overwrite', type = str2bool, default = False)
    parser.add_argument('--streaming', type = str2bool, default = True)
    parser.add_argument('--data_dir', type=str, default=None)
    parser.add_argument('--shuffle', type = str2bool, default = True)
    parser.add_argument('--task', type = str, default = None)
    parser.add_argument('--hf_use_auth_token', type = str, default = None,
                        help = "Huggingface auth token for downloading datasets; Load from env var if not provided.")
    
    args = parser.parse_args()
    if args.hf_use_auth_token is None:
        args.hf_use_auth_token = os.environ.get("HF_USE_AUTH_TOKEN", None)

    print(args)
    main(args)
tigerscore/download_dataset/download_general_datasets.py
TIGER-AI-Lab/TIGERScore
import hashlib
import argparse 

def generate_hash_code(text):
    # Convert the text to bytes and create a hash object
    hash_object = hashlib.sha256(text.encode())

    # Get the hexadecimal representation of the hash code
    hex_code = hash_object.hexdigest()

    # Return the first 16 digits of the hexadecimal code
    return hex_code[:16]

def str2bool(v):
    if isinstance(v, bool):
        return v
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

def empty2None(x):
    if x == '':
        return None
    else:
        return x

def empty2zero(x):
    if x == '':
        return 0
    elif isinstance(x, int):
        return x
    elif isinstance(x, str):
        return int(x)
    else:
        raise argparse.ArgumentTypeError('Integer value expected.')

tigerscore/download_dataset/utils.py
TIGER-AI-Lab/TIGERScore
# %%
from bs_utils import *
from copy import deepcopy
from tqdm import trange
from tqdm import tqdm


class SUMStat:
    """ A class used to get stats of SUM trained data """

    def __init__(self, path):
        self.path = path
        self.data = read_pickle(path)
        self.sample_id = list(self.data.keys())[0]
        self.sample_sys = list(self.data[self.sample_id]['sys_summs'].keys())[0]
        self._metrics = list(self.data[self.sample_id]['sys_summs'][self.sample_sys]['scores'].keys())
        self._auto_metrics = [x for x in self.metrics if x not in self.human_metrics]

    def save_data(self, path=None):
        if path is None:
            path = self.path
        save_pickle(self.data, path)

    def evaluate_summary(self, human_metric, auto_metrics=None, table=None):
        """ Evaluate summaries. Conduct summary-level correlations w.r.t each document """
        assert human_metric in self.human_metrics
        if auto_metrics is None:
            auto_metrics = self.auto_metrics
        print(f'Human metric: {human_metric}')
        headers = ['metric', 'spearman', 'kendalltau']
        metric_with_corr = []
        for metric in auto_metrics:
            correlations = []
            for doc_id in self.data:
                target_scores = []
                prediction_scores = []

                sys_summs = self.data[doc_id]['sys_summs']
                for sys_name in sys_summs:
                    prediction_scores.append(sys_summs[sys_name]['scores'][metric])
                    target_scores.append(sys_summs[sys_name]['scores'][human_metric])
                if len(set(prediction_scores)) == 1 or len(set(target_scores)) == 1:
                    continue
                correlations.append([spearmanr(target_scores, prediction_scores)[0],
                                     kendalltau(target_scores, prediction_scores)[0]])
            corr_mat = np.array(correlations)
            spearman, ktau = np.mean(corr_mat[:, 0]), np.mean(corr_mat[:, 1])
            metric_with_corr.append([metric, spearman, ktau])
        sorted_metric_with_corr = sorted(metric_with_corr, key=lambda x: x[1], reverse=True)
        if table is not None:
            file = open(table, 'w')
            for each in sorted_metric_with_corr:
                print(f'{each[0]}\t{each[1]}\t{each[2]}', file=file)
            file.flush()
        print(tabulate(sorted_metric_with_corr, headers=headers, tablefmt='simple'))

    def get_fact_pearson(self, auto_metrics=None):
        assert 'QAGS' in self.path
        headers = ['metric', 'pearson']
        metric_with_corr = []
        if auto_metrics is None:
            auto_metrics = self.auto_metrics
        for metric in auto_metrics:
            human_scores = []
            metric_scores = []
            for doc_id in self.data:
                human_scores.append(self.data[doc_id]['sys_summs'][0]['scores']['fact'])
                metric_scores.append(self.data[doc_id]['sys_summs'][0]['scores'][metric])
            pearson, _ = pearsonr(human_scores, metric_scores)
            metric_with_corr.append([metric, pearson])
        metric_with_corr = sorted(metric_with_corr, key=lambda x: x[1], reverse=True)
        print(tabulate(metric_with_corr, headers=headers, tablefmt='simple'))

    def fact_pearson_sig_test(self, metric_list):
        for m in metric_list:
            assert m in self.auto_metrics
        comp_tab = np.zeros((len(metric_list), len(metric_list)), dtype=int)
        for i in range(len(metric_list)):  # row
            for j in range(i + 1, len(metric_list)):  # col
                m1 = metric_list[i]
                m2 = metric_list[j]
                # Test if m1 is significant better than m2
                out = self.fact_pearson_sig_test_two(m1, m2)
                if out == 1:
                    comp_tab[j][i] = 1
                elif out == -1:
                    comp_tab[i][j] = 1
                else:
                    pass
        result = comp_tab.sum(axis=1)
        best_metrics = []
        for i in range(len(result)):
            if result[i] == 0:
                best_metrics.append(metric_list[i])
        print(f'Best metrics are: {best_metrics}')

    def fact_pearson_sig_test_two(self, m1, m2):
        assert 'QAGS' in self.path
        random.seed(666)
        doc_ids = list(self.data.keys())
        better = 0
        for i in trange(1000):
            random.shuffle(doc_ids)
            sub_ids = doc_ids[:int(0.8 * len(doc_ids))]
            m1_scores, m2_scores, human_scores = [], [], []

            for doc_id in sub_ids:
                human_scores.append(self.data[doc_id]['sys_summs'][0]['scores']['fact'])
                m1_scores.append(self.data[doc_id]['sys_summs'][0]['scores'][m1])
                m2_scores.append(self.data[doc_id]['sys_summs'][0]['scores'][m2])
            pearson_m1, _ = pearsonr(human_scores, m1_scores)
            pearson_m2, _ = pearsonr(human_scores, m2_scores)
            if pearson_m1 > pearson_m2:
                better += 1
        if better > 950:
            return 1
        elif better < 50:
            return -1
        else:
            return 0

    def get_fact_acc(self, auto_metrics=None):
        """ Used for the Rank19 dataset. """
        assert 'Rank' in self.path
        headers = ['metric', 'acc']
        metric_with_acc = []
        if auto_metrics is None:
            auto_metrics = self.auto_metrics
        for metric in auto_metrics:
            correct = 0
            for doc_id in self.data:
                if self.data[doc_id]['sys_summs']['correct']['scores'][metric] > \
                        self.data[doc_id]['sys_summs']['incorrect']['scores'][metric]:
                    correct += 1
            metric_with_acc.append([metric, correct / len(self.data)])
        metric_with_acc = sorted(metric_with_acc, key=lambda x: x[1], reverse=True)
        print(tabulate(metric_with_acc, headers=headers, tablefmt='simple'))

    def fact_acc_sig_test(self, metric_list):
        for m in metric_list:
            assert m in self.auto_metrics
        comp_tab = np.zeros((len(metric_list), len(metric_list)), dtype=int)
        for i in range(len(metric_list)):  # row
            for j in range(i + 1, len(metric_list)):  # col
                m1 = metric_list[i]
                m2 = metric_list[j]
                # Test if m1 is significant better than m2
                out = self.fact_acc_sig_test_two(m1, m2)
                if out == 1:
                    comp_tab[j][i] = 1
                elif out == -1:
                    comp_tab[i][j] = 1
                else:
                    pass
        result = comp_tab.sum(axis=1)
        best_metrics = []
        for i in range(len(result)):
            if result[i] == 0:
                best_metrics.append(metric_list[i])
        print(f'Best metrics are: {best_metrics}')

    def fact_acc_sig_test_two(self, m1, m2):
        """ Return 1 if m1 significant better than m2, or -1 if m1 significant worse than m2
            or 0 if cannot decide.
        """
        assert 'Rank' in self.path
        random.seed(666)
        doc_ids = list(self.data.keys())
        better = 0
        for i in trange(1000):
            random.shuffle(doc_ids)
            sub_ids = doc_ids[:int(0.8 * len(doc_ids))]
            m1_correct = 0
            m2_correct = 0
            for doc_id in sub_ids:
                if self.data[doc_id]['sys_summs']['correct']['scores'][m1] > \
                        self.data[doc_id]['sys_summs']['incorrect']['scores'][m1]:
                    m1_correct += 1
                if self.data[doc_id]['sys_summs']['correct']['scores'][m2] > \
                        self.data[doc_id]['sys_summs']['incorrect']['scores'][m2]:
                    m2_correct += 1
            if m1_correct > m2_correct:
                better += 1
        if better > 950:
            return 1
        elif better < 50:
            return -1
        else:
            return 0

    def sig_test(self, metric_list, human_metric):
        """ Comparisons between all pairs of metrics. Using Spearman correlation. """
        for m in metric_list:
            assert m in self.auto_metrics
        comp_tab = np.zeros((len(metric_list), len(metric_list)), dtype=int)
        for i in range(len(metric_list)):  # row
            for j in range(i + 1, len(metric_list)):  # col
                m1 = metric_list[i]
                m2 = metric_list[j]
                # Test if m1 is significant better than m2
                out = self.sig_test_two(m1, m2, human_metric)
                if out == 1:
                    comp_tab[j][i] = 1
                elif out == -1:
                    comp_tab[i][j] = 1
                else:
                    pass
        result = comp_tab.sum(axis=1)
        best_metrics = []
        for i in range(len(result)):
            if result[i] == 0:
                best_metrics.append(metric_list[i])
        print(f'Best metrics are: {best_metrics}')

    def sig_test_two(self, m1, m2, human_metric):
        """ Comparisons between a pair of metrics. Using Spearman correlation.
            Test if m1 is significant better than m2. return 1 if m1 is better,
            return -1 if m2 is better, otherwise return 0
        """
        assert (not 'Rank' in self.path) and (not 'QAGS' in self.path)
        random.seed(666)
        doc_ids = list(self.data.keys())
        better = 0
        for i in trange(1000):
            random.shuffle(doc_ids)
            sub_ids = doc_ids[:int(0.8 * len(doc_ids))]
            corr1, corr2 = [], []
            for doc_id in sub_ids:
                target, pred1, pred2 = [], [], []
                sys_summs = self.data[doc_id]['sys_summs']
                for sys_name in sys_summs:
                    pred1.append(sys_summs[sys_name]['scores'][m1])
                    pred2.append(sys_summs[sys_name]['scores'][m2])
                    target.append(sys_summs[sys_name]['scores'][human_metric])
                if len(set(pred1)) == 1 or len(set(pred2)) == 1 or len(set(target)) == 1:
                    continue
                corr1.append(spearmanr(target, pred1)[0])
                corr2.append(spearmanr(target, pred2)[0])

            corr1 = np.mean(corr1)
            corr2 = np.mean(corr2)
            if corr1 > corr2:
                better += 1
        if better > 950:
            return 1
        elif better < 50:
            return -1
        else:
            return 0

    def combine_prompt(self):
        """ Take the average of all prompted results for a single prediction.
            We consider encoder-based prompts and decoder-based prompts separately.
        """

        def get_keys(s):
            """ Get the first key and second key in MAP """
            k1, k2 = None, None
            if s.startswith('bart_score_cnn'):
                k1 = 'bart_score_cnn'
            elif s.startswith('bart_score_para'):
                k1 = 'bart_score_para'
            else:
                k1 = 'bart_score'
            if 'src' in s:
                if '_en_' in s:
                    k2 = 'src_hypo_en'
                else:
                    k2 = 'src_hypo_de'
            if 'hypo_ref' in s:
                if '_en_' in s:
                    k2 = 'hypo_ref_en'
                else:
                    k2 = 'hypo_ref_de'
            if 'ref_hypo' in s:
                if '_en_' in s:
                    k2 = 'ref_hypo_en'
                else:
                    k2 = 'ref_hypo_de'
            if 'avg_f' in s:
                if '_en_' in s:
                    k2 = 'avg_f_en'
                else:
                    k2 = 'avg_f_de'
            if 'harm_f' in s:
                if '_en_' in s:
                    k2 = 'harm_f_en'
                else:
                    k2 = 'harm_f_de'
            return k1, k2

        for doc_id in self.data:
            sys_summs = self.data[doc_id]['sys_summs']
            for sys_name in sys_summs:
                types = {
                    'src_hypo_en': [],
                    'src_hypo_de': [],
                    'ref_hypo_en': [],
                    'ref_hypo_de': [],
                    'hypo_ref_en': [],
                    'hypo_ref_de': [],
                    'avg_f_en': [],
                    'avg_f_de': [],
                    'harm_f_en': [],
                    'harm_f_de': []
                }
                MAP = {
                    'bart_score': deepcopy(types),
                    'bart_score_cnn': deepcopy(types),
                    'bart_score_para': deepcopy(types)
                }
                scores = sys_summs[sys_name]['scores']
                for k in scores:
                    if '_en_' in k or '_de_' in k:
                        k1, k2 = get_keys(k)
                        MAP[k1][k2].append(scores[k])
                for k, v in MAP.items():
                    for kk, vv in v.items():
                        if len(vv) == 0:
                            continue
                        new_m = k + '_' + kk
                        if new_m not in self.auto_metrics:
                            print(f'new_metric: {new_m}')
                            self._metrics.append(new_m)
                            self._auto_metrics.append(new_m)
                        self.data[doc_id]['sys_summs'][sys_name]['scores'][new_m] = sum(vv) / len(vv)

    @property
    def auto_metrics(self):
        return self._auto_metrics

    @property
    def metrics(self):
        return self._metrics

    @property
    def human_metrics(self):
        """ All available human metrics. """
        if 'REALSumm' in self.path:
            return ['litepyramid_recall']
        if 'SummEval' in self.path:
            return ['coherence', 'consistency', 'fluency', 'relevance']
        if 'Newsroom' in self.path:
            return ['coherence', 'fluency', 'informativeness', 'relevance']
        if 'Rank19' in self.path or 'QAGS' in self.path:
            return ['fact']


class D2TStat:
    """ A class used to get stats of D2T trained data """

    def __init__(self, path):
        self.path = path
        self.data = read_pickle(path)
        self.sample_id = list(self.data.keys())[0]
        self._metrics = list(self.data[self.sample_id]['scores'].keys())
        self._auto_metrics = [x for x in self.metrics if x not in self.human_metrics]

    def evaluate_text(self, human_metric, auto_metrics=None, table=None):
        print(f'Human metric: {human_metric}')
        headers = ['metric', 'spearman', 'kendalltau']
        metric_with_corr = []
        if auto_metrics is None:
            auto_metrics = self.auto_metrics
        for metric in auto_metrics:
            human_scores = []
            metric_scores = []
            for doc_id in self.data:
                human_scores.append(self.data[doc_id]['scores'][human_metric])
                metric_scores.append(self.data[doc_id]['scores'][metric])
            spearman = spearmanr(human_scores, metric_scores)[0]
            ktau = kendalltau(human_scores, metric_scores)[0]
            metric_with_corr.append([metric, spearman, ktau])
        sorted_metric_with_corr = sorted(metric_with_corr, key=lambda x: x[1], reverse=True)
        if table is not None:
            file = open(table, 'w')
            for each in sorted_metric_with_corr:
                print(f'{each[0]}\t{each[1]}\t{each[2]}', file=file)
            file.flush()
        print(tabulate(sorted_metric_with_corr, headers=headers, tablefmt='simple'))

    def sig_test_two(self, m1, m2, human_metric):
        human_scores = []
        m1_scores = []
        m2_scores = []
        doc_ids = list(self.data.keys())
        better = 0
        random.seed(666)
        for i in trange(1000):
            random.shuffle(doc_ids)
            sub_ids = doc_ids[:int(0.8 * len(doc_ids))]
            for doc_id in sub_ids:
                human_scores.append(self.data[doc_id]['scores'][human_metric])
                m1_scores.append(self.data[doc_id]['scores'][m1])
                m2_scores.append(self.data[doc_id]['scores'][m2])
            spearman1, _ = spearmanr(human_scores, m1_scores)
            spearman2, _ = spearmanr(human_scores, m2_scores)
            if spearman1 > spearman2:
                better += 1
        if better > 950:
            return 1
        elif better < 50:
            return -1
        else:
            return 0

    def combine_prompt(self):
        def get_keys(s):
            """ Get the first key and second key in MAP """
            k1, k2 = None, None
            if s.startswith('bart_score_cnn'):
                k1 = 'bart_score_cnn'
            elif s.startswith('bart_score_para'):
                k1 = 'bart_score_para'
            else:
                k1 = 'bart_score'
            if 'src' in s:
                if '_en_' in s:
                    k2 = 'src_hypo_en'
                else:
                    k2 = 'src_hypo_de'
            if 'hypo_ref' in s:
                if '_en_' in s:
                    k2 = 'hypo_ref_en'
                else:
                    k2 = 'hypo_ref_de'
            if 'ref_hypo' in s:
                if '_en_' in s:
                    k2 = 'ref_hypo_en'
                else:
                    k2 = 'ref_hypo_de'
            if 'avg_f' in s:
                if '_en_' in s:
                    k2 = 'avg_f_en'
                else:
                    k2 = 'avg_f_de'
            if 'harm_f' in s:
                if '_en_' in s:
                    k2 = 'harm_f_en'
                else:
                    k2 = 'harm_f_de'
            return k1, k2

        for doc_id in self.data:
            types = {
                'src_hypo_en': [],
                'src_hypo_de': [],
                'ref_hypo_en': [],
                'ref_hypo_de': [],
                'hypo_ref_en': [],
                'hypo_ref_de': [],
                'avg_f_en': [],
                'avg_f_de': [],
                'harm_f_en': [],
                'harm_f_de': []
            }
            MAP = {
                'bart_score': deepcopy(types),
                'bart_score_cnn': deepcopy(types),
                'bart_score_para': deepcopy(types)
            }
            scores = self.data[doc_id]['scores']
            for k in scores:
                if '_en_' in k or '_de_' in k:
                    k1, k2 = get_keys(k)
                    MAP[k1][k2].append(scores[k])
            for k, v in MAP.items():
                for kk, vv in v.items():
                    if len(vv) == 0:
                        continue
                    new_m = k + '_' + kk
                    if new_m not in self.auto_metrics:
                        print(f'new_metric: {new_m}')
                        self._metrics.append(new_m)
                        self._auto_metrics.append(new_m)
                    self.data[doc_id]['scores'][new_m] = sum(vv) / len(vv)

    def save_data(self, path=None):
        if path is None:
            path = self.path
        save_pickle(self.data, path)

    @property
    def auto_metrics(self):
        return self._auto_metrics

    @property
    def metrics(self):
        return self._metrics

    @property
    def human_metrics(self):
        return ['informativeness', 'naturalness', 'quality']


class WMTStat:
    """ A class used to get stats of WMT trained data """

    def __init__(self, path):
        self.path = path
        self.data = read_pickle(path)
        self._metrics = list(self.data[0]['better']['scores'].keys())
        pos = path.find('-en')
        self.lp = path[pos - 2: pos + 3]
        # systems ranked by their DA score
        self._systems = {
            'de-en': ['Facebook_FAIR.6750', 'RWTH_Aachen_System.6818', 'MSRA.MADL.6910', 'online-B.0', 'JHU.6809',
                      'MLLP-UPV.6899', 'dfki-nmt.6478', 'UCAM.6461', 'online-A.0', 'NEU.6801', 'uedin.6749',
                      'online-Y.0', 'TartuNLP-c.6502', 'online-G.0', 'PROMT_NMT_DE-EN.6683', 'online-X.0'],
            'fi-en': ['MSRA.NAO.6983', 'online-Y.0', 'GTCOM-Primary.6946', 'USYD.6995', 'online-B.0',
                      'Helsinki_NLP.6889', 'online-A.0', 'online-G.0', 'TartuNLP-c.6905', 'online-X.0', 'parfda.6526',
                      'apertium-fin-eng-unconstrained-fien.6449'],
            'gu-en': ['NEU.6756', 'UEDIN.6534', 'GTCOM-Primary.6969', 'CUNI-T2T-transfer-guen.6431',
                      'aylien_mt_gu-en_multilingual.6826', 'NICT.6603', 'online-G.0', 'IITP-MT.6824', 'UdS-DFKI.6861',
                      'IIITH-MT.6688', 'Ju_Saarland.6525'],
            'kk-en': ['online-B.0', 'NEU.6753', 'rug_kken_morfessor.6677', 'online-G.0', 'talp_upc_2019_kken.6657',
                      'NRC-CNRC.6895', 'Frank_s_MT.6127', 'NICT.6770', 'CUNI-T2T-transfer-kken.6436', 'UMD.6736',
                      'DBMS-KU_KKEN.6726'],
            'lt-en': ['GTCOM-Primary.6998', 'tilde-nc-nmt.6881', 'NEU.6759', 'MSRA.MASS.6945', 'tilde-c-nmt.6876',
                      'online-B.0', 'online-A.0', 'TartuNLP-c.6908', 'online-G.0', 'JUMT.6616', 'online-X.0'],
            'ru-en': ['Facebook_FAIR.6937', 'online-G.0', 'eTranslation.6598', 'online-B.0', 'NEU.6803',
                      'MSRA.SCA.6976', 'rerank-re.6540', 'online-Y.0', 'online-A.0', 'afrl-syscomb19.6782',
                      'afrl-ewc.6659', 'TartuNLP-u.6650', 'online-X.0', 'NICT.6561'],
            'zh-en': ['Baidu-system.6940', 'KSAI-system.6927', 'MSRA.MASS.6996', 'MSRA.MASS.6942', 'NEU.6832',
                      'BTRANS.6825', 'online-B.0', 'BTRANS-ensemble.6992', 'UEDIN.6530', 'online-Y.0', 'NICT.6814',
                      'online-A.0', 'online-G.0', 'online-X.0', 'Apprentice-c.6706']
        }

    def save_data(self, path=None):
        if path is None:
            path = self.path
        save_pickle(self.data, path)

    def retrieve_scores(self, metric, doc_ids):
        """ retrieve better, worse scores """
        better, worse = [], []
        for doc_id in doc_ids:
            better.append(float(self.data[doc_id]['better']['scores'][metric]))
            worse.append(float(self.data[doc_id]['worse']['scores'][metric]))
        return better, worse

    def kendall(self, hyp1_scores: list, hyp2_scores: list):
        """ Computes the official WMT19 shared task Kendall correlation score. """
        assert len(hyp1_scores) == len(hyp2_scores)
        conc, disc = 0, 0

        for x1, x2 in zip(hyp1_scores, hyp2_scores):
            if x1 > x2:
                conc += 1
            else:
                disc += 1
        return (conc - disc) / (conc + disc)

    def print_ktau(self, metrics=None):
        headers = ['metric', 'k-tau']
        metric_with_ktau = []
        doc_ids = list(self.data.keys())
        if metrics is None:
            metrics = self.metrics
        for metric in tqdm(metrics):
            better, worse = self.retrieve_scores(metric, doc_ids)
            ktau = self.kendall(better, worse)
            metric_with_ktau.append([metric, ktau])
        sorted_metric_with_ktau = sorted(metric_with_ktau, key=lambda x: x[1], reverse=True)
        print(tabulate(sorted_metric_with_ktau, headers=headers, tablefmt='simple'))

    def print_ref_len(self):
        """ Get the length of reference texts """
        ref_lens = []
        for doc_id in self.data:
            ref = self.data[doc_id]['ref']
            ref_len = len(ref.split(' '))
            ref_lens.append(ref_len)
        print(f'Mean reference length: {np.mean(ref_lens)}')
        print(f'Max reference length: {np.max(ref_lens)}')
        print(f'Min reference length: {np.min(ref_lens)}')
        print(f'20% percentile: {np.percentile(ref_lens, 20)}')
        print(f'80% percentile: {np.percentile(ref_lens, 80)}')
        print(f'90% percentile: {np.percentile(ref_lens, 90)}')

    def print_len_ktau(self, min_len, max_len, metrics=None):
        headers = ['metric', 'k-tau']
        metric_with_ktau = []
        sub_ids = []
        for doc_id in tqdm(self.data):
            ref_len = len(self.data[doc_id]['ref'].split(' '))
            if min_len <= ref_len <= max_len:
                sub_ids.append(doc_id)
        print(f'Considered samples: {len(sub_ids)}')
        if metrics is None:
            metrics = self.metrics
        for metric in tqdm(metrics):
            better, worse = self.retrieve_scores(metric, sub_ids)
            ktau = self.kendall(better, worse)
            metric_with_ktau.append([metric, ktau])
        sorted_metric_with_ktau = sorted(metric_with_ktau, key=lambda x: x[1], reverse=True)
        print(tabulate(sorted_metric_with_ktau, headers=headers, tablefmt='simple'))

    def sig_test_two(self, m1, m2):
        random.seed(666)
        doc_ids = list(self.data.keys())
        better = 0
        for _ in trange(1000):
            random.shuffle(doc_ids)
            sub_ids = doc_ids[:int(0.8 * len(doc_ids))]
            better_m1, worse_m1, better_m2, worse_m2 = [], [], [], []
            for doc_id in sub_ids:
                better_m1.append(float(self.data[doc_id]['better']['scores'][m1]))
                worse_m1.append(float(self.data[doc_id]['worse']['scores'][m1]))
                better_m2.append(float(self.data[doc_id]['better']['scores'][m2]))
                worse_m2.append(float(self.data[doc_id]['worse']['scores'][m2]))
            m1_ktau = self.kendall(better_m1, worse_m1)
            m2_ktau = self.kendall(better_m2, worse_m2)
            if m1_ktau > m2_ktau:
                better += 1

        if better > 950:
            return 1
        elif better < 50:
            return -1
        else:
            return 0

    @property
    def metrics(self):
        return self._metrics

    @property
    def systems(self):
        return self._systems[self.lp]

tigerscore/eval_scripts/bs_analysis.py
TIGER-AI-Lab/TIGERScore
import pickle
import jsonlines
import nltk
from nltk.tokenize import sent_tokenize
from nltk import word_tokenize
import numpy as np
from tabulate import tabulate
from mosestokenizer import *
import random
from random import choices
import os
import sys
import re
import json
from collections import defaultdict as ddict
from scipy.stats import pearsonr, spearmanr, kendalltau
import logging

# nltk.download('stopwords')
detokenizer = MosesDetokenizer('en')

def save_json(file_name, data):
    with open(file_name, 'w') as f:
        json.dump(data, f, indent=4, ensure_ascii=False)

def load_json(file_name):
    with open(file_name, 'r') as f:
        return json.load(f)

def read_pickle(file):
    with open(file, 'rb') as f:
        data = pickle.load(f)
    return data


def save_pickle(file,data):
    with open(file, 'wb') as f:
        pickle.dump(data, f)
    print(f'Saved to {file}.')


def read_file_to_list(file_name):
    lines = []
    with open(file_name, 'r', encoding='utf8') as f:
        for line in f.readlines():
            lines.append(line.strip())
    return lines


def write_list_to_file(list_to_write, filename):
    out_file = open(filename, 'w')
    for line in list_to_write:
        print(line, file=out_file)
    out_file.flush()
    out_file.close()
    print(f'Saved to {filename}.')


def read_jsonlines_to_list(file_name):
    lines = []
    with jsonlines.open(file_name, 'r') as reader:
        for obj in reader:
            lines.append(obj)
    return lines


def write_list_to_jsonline(list_to_write, filename):
    with jsonlines.open(filename, 'w') as writer:
        writer.write_all(list_to_write)
    print(f'Saved to {filename}.')


def capitalize_sents(text: str):
    """ Given a string, capitalize the initial letter of each sentence. """
    sentences = sent_tokenize(text)
    sentences = [sent.strip() for sent in sentences]
    sentences = [sent.capitalize() for sent in sentences]
    sentences = " ".join(sentences)
    return sentences


def is_capitalized(text: str):
    """ Given a string (system output etc.) , check whether it is lowercased,
        or normally capitalized.
    """
    return not text.islower()


def tokenize(text: str):
    words = word_tokenize(text)
    return " ".join(words)


def detokenize(text: str):
    words = text.split(" ")
    return detokenizer(words)


def use_original_bracket(text: str):
    return text.replace('-lrb-', '(').replace('-rrb-', ')').replace('-LRB-', '(').replace('-RRB-', ')').replace('-lsb-',
                                                                                                                '[').replace(
        '-rsb-', ']').replace('-LSB-', '[').replace('-RSB-', ']')


# Disable print
def blockPrint():
    sys.stdout = open(os.devnull, 'w')


# Restore print
def enablePrint():
    sys.stdout = sys.__stdout__


def retrieve_scores(saveto):
    def get_r_p_f(line):
        line = line.split(" ")
        r = float(line[-3][-7:])
        p = float(line[-2][-7:])
        f = float(line[-1][-7:])
        return r, p, f

    lines = read_file_to_list(saveto)
    rouge1_list, rouge2_list, rougel_list = [], [], []
    for line in lines:
        if line.startswith('1 ROUGE-1 Eval'):
            rouge1_list.append(get_r_p_f(line))
        if line.startswith('1 ROUGE-2 Eval'):
            rouge2_list.append(get_r_p_f(line))
        if line.startswith('1 ROUGE-L Eval'):
            rougel_list.append(get_r_p_f(line))
    return rouge1_list, rouge2_list, rougel_list


def get_rank(data, metric):
    """ Rank all systems based on a metric (avg score) """
    scores = {}  # {sysname: [scores]}
    for doc_id in data:
        sys_summs = data[doc_id]['sys_summs']
        for sys_name in sys_summs:
            score = sys_summs[sys_name]['scores'][metric]
            scores.setdefault(sys_name, []).append(score)
    scores = {k: sum(v) / len(v) for k, v in scores.items()}
    new_scores = dict(sorted(scores.items(), key=lambda x: x[1], reverse=True))
    # for k in new_scores:
    #     print(k)
    return new_scores.keys()


def get_sents_from_tags(text, sent_start_tag, sent_end_tag):
    sents = re.findall(r'%s (.+?) %s' % (sent_start_tag, sent_end_tag), text)
    sents = [sent for sent in sents if len(sent) > 0]  # remove empty sents
    return sents


def get_metrics_list(sd):
    """
    Does each system summary dict have same all_metrics?
    :param sd: scores dict
    :return: list of all_metrics in the scores dict
    """
    metrics_tuple_set = set(
        [tuple(sorted(list(x['scores'].keys())))
         for d in sd.values() for x in d['sys_summs'].values()])
    assert len(metrics_tuple_set) == 1, (
        metrics_tuple_set, "all system summary score dicts should have the same set of all_metrics")
    metrics_list = list(list(metrics_tuple_set)[0])
    return metrics_list


def print_score_ranges(sd):
    metrics_list = get_metrics_list(sd)
    print_list = []
    headers = ["min", "25-perc", "median", "75-perc", "max", "mean"]
    for m in metrics_list:
        scores = [s['scores'][m] for d in sd.values() for s in d['sys_summs'].values() if s['sys_summ'] != 'EMPTY']
        print_list.append([m,
                           np.min(scores),
                           np.percentile(scores, 25),
                           np.median(scores),
                           np.percentile(scores, 75),
                           np.max(scores),
                           np.mean(scores)])
    print(tabulate(print_list, headers=headers, floatfmt=".6f", tablefmt="simple"))


def get_system_level_scores(sd, metrics, agg='mean', nas=False):
    """
    systems[system_name][metric] = average_score or list of scores
    """
    systems = ddict(lambda: ddict(list))

    for isd in sd.values():
        for system_name, scores in isd['sys_summs'].items():
            for m in metrics:
                # Empty summary
                if scores['sys_summ'] == 'EMPTY':
                    systems[system_name][m].append(None)
                else:
                    systems[system_name][m].append(scores['scores'][m])

    for system_name, scores in systems.items():
        for m in scores:
            all_scores = systems[system_name][m]
            if agg == 'mean':
                all_scores = [x for x in all_scores if x is not None]
                systems[system_name][m] = np.mean(all_scores)

    if nas:
        min_scores = {}
        max_scores = {}
        for m in metrics:
            min_scores[m] = np.min([systems[sys][m] for sys in systems.keys()])
            max_scores[m] = np.max([systems[sys][m] for sys in systems.keys()])
        for sys in systems:
            systems[sys]['nas'] = np.mean([
                (systems[sys][m] - min_scores[m]) / (max_scores[m] - min_scores[m]) for m in metrics
            ])

    return systems


def get_topk(systems, k, metric='rouge2_f'):
    systems_l = [(name, score[metric]) for name, score in systems.items()]
    systems_l = sorted(systems_l, key=lambda x: x[1], reverse=True)
    topk_system_names = [tup[0] for tup in systems_l[:k]]
    return {name: systems[name] for name in topk_system_names}


def print_correlation(topk_systems, metric_pairs):
    # disagreement between every pair of metrics for the topk
    headers = ['metric_pair', 'pearson', 'spearman', 'kendalltau']
    print_list = []
    for pair in metric_pairs:
        if 'bart_en_sim' in pair[1] or 'bart_sim' in pair[1]:
            continue
        m1_scores = []
        m2_scores = []
        for scores in topk_systems.values():
            m1_scores.append(scores[pair[0]])  # Human metric
            m2_scores.append(scores[pair[1]])

        pearson, _ = pearsonr(m1_scores, m2_scores)
        spearman, _ = spearmanr(m1_scores, m2_scores)
        ktau, _ = kendalltau(m1_scores, m2_scores)
        print_list.append([f'{pair[1]}', pearson, spearman, ktau])
    # sort based on pearson
    print_list = sorted(print_list, key=lambda x: x[2], reverse=True)
    print(tabulate(print_list, headers=headers, tablefmt='simple'))


def get_predictions_br(system_pairs, systems, metric):
    random.seed(666)
    preds = {}
    for pair in system_pairs:
        sys1 = systems[pair[0]][metric]
        sys2 = systems[pair[1]][metric]
        n = len(sys1)
        points = [i for i in range(0, n)]
        is_better = 0
        N = 1000
        for i in range(N):
            sample = choices(points, k=n)
            sys1_, sys2_ = [], []
            # Due to EMPTY summary, we have to ensure sys1_, sys2_ not empty
            while len(sys1_) == 0:
                for p in sample:
                    if sys1[p] is None or sys2[p] is None:
                        continue
                    else:
                        sys1_.append(sys1[p])
                        sys2_.append(sys2[p])
                sample = choices(points, k=n)
            if np.mean(sys1_) > np.mean(sys2_):
                is_better += 1

        if is_better / N >= 0.95:
            preds[pair] = 0  # pair[0] is better
        elif is_better / N <= 0.05:
            preds[pair] = 1  # pair[1] is better
        else:
            preds[pair] = 2  # can't say
    return preds

def reformat_sum_for_bartscore(data_path,our_score_name):
    logging.info(f"Reformatting {data_path} to bartscore format")
    data = load_json(data_path)
    new_data = {}

    for item in data:
        id = item["id"]
        input = item["input"]
        output = item["output"]
        candidates = item["candidates"]
        new_data_item = {
            "src": input,
            "ref_summ": output,
            "sys_summs": {}
        }
        good = True
        for cand in candidates:
            model = cand["model"]
            text = cand["text"]
            scores = cand["scores"]
            if our_score_name not in scores:
                print("xgptscore")
                good = False
                break
            new_data_item["sys_summs"][model] = {
                "sys_summ": text,
                "scores": scores
            }
        if good:
            new_data[id] = new_data_item
    
    logging.info("Total good data: {}/{}".format(len(new_data), len(data)))
    new_data_path = data_path.with_name(data_path.stem + "_bs_format.pkl")
    save_pickle(new_data_path, new_data)
    save_json(new_data_path.with_suffix(".json"), new_data)
    logging.info("Saved data in bartscore format to {}".format(new_data_path))
    
def reformat_d2t_for_bartscore(data_path, our_score_name):
    logging.info(f"Reformatting {data_path} to bartscore format")
    data = load_json(data_path)
    new_data = {}

    for item in data:
        id = item["id"]
        input = item["input"]
        output = item["output"]
        candidates = item["candidates"]
        if our_score_name not in candidates[0]["scores"] or candidates[0]["scores"][our_score_name] is None:
            continue
        scores = candidates[0]["scores"]
        new_data[id] = {
            "src": input,
            "ref_summs": output,
            "sys_summ": candidates[0]["text"],
            "scores": scores
        }

    logging.info("Total good data: {}/{}".format(len(new_data), len(data)))
    new_data_path = data_path.with_name(data_path.stem + "_bs_format.pkl")
    save_pickle(new_data_path, new_data)
    logging.info("Saved data in bartscore format to {}".format(new_data_path))
    
tigerscore/eval_scripts/bs_utils.py
TIGER-AI-Lab/TIGERScore
import sys
import fire
import json
import logging
import regex as re
import copy
import random
sys.path.append("..")
from xgptscore.openai_utils import openai_completions, _chatml_to_prompt
from typing import List, Dict
from string import Template
from collections import Counter
logging.basicConfig(level=logging.INFO)

# check_template = """
# Instruction: ${instruction}
# Source: ${source}
# Model-generated output: ${model_generated}
# Identified errors in the model-generated output: 
# ${identified_errors}

# For each error, answer the following questions:
# Q1: Extracted pointed error phrase and the correction phrase in the explantion text. The error phrase is a substr in the model-generated output and the correction phrase is the correction of it in the explnation. The Fill null in the answer if error phrase or correction phrase is not found. (Answer [str, str]).
# A1:
# Q2: Does this error indeed occur in the model-generated output? An error might be (Answer Yes/No)
# A2:
# Q3: If A1 contains both the error phrase and the correction phrase, is the correction actually fixing the error considering the instruction and source input? (Answer Yes/No)
# A3:
# Q4: Is the reduction score appropriate considering the severity and the sentence context? (Answer Yes/No)
# A4: 
# Q5: Does the explanation provide a helpful content for understanding the error and how to fix it? (Answer Yes/No)
# A5:
# Q6: Does the explanation contain some hallucinated content outside the provided instruction, source input, and model-generated output? (Answer Yes/No)
# A6:
# The output format will be in JSON
# {
# error_1: {Q1: A1, Q2: A2, Q3: A3, Q4: A4, Q5: A5, Q6: A6}, 
# ...
# }
# """

check_template = """
Instruction: ${instruction}
Source: ${source}
Model-generated output: ${model_generated}
Identified errors in the model-generated output: 
${identified_errors}

For each error, answer the following questions:
Q1: Extracted pointed error phrase and the correction phrase in the explantion text. The error phrase is a substr in the model-generated output and the correction phrase is the correction of it in the explnation. The Fill null in the answer if error phrase or correction phrase is not found. (Answer [str, str]).
A1:
Q2: Does this error indeed occur in the model-generated output? An error might be (Answer Yes/No)
A2:
Q3: If A1 contains both the error phrase and the correction phrase, is the correction actually fixing the error considering the instruction and source input? (Answer Yes/No)
A3:
Q4: Is the reduction score appropriate considering the severity and the sentence context? (Answer Yes/No)
A4: 
Q5: Does the explanation provide a helpful content for understanding the error and how to fix it? (Answer Yes/No)
A5:
Q6: Does the explanation contain some hallucinated content outside the provided instruction, source input, and model-generated output? (Answer Yes/No)
A6:
The output format will be in JSON
{
error_1: {Q1: A1, Q2: A2, Q3: A3, Q4: A4, Q5: A5, Q6: A6}, 
...
}
"""

# check_template = """
# Instruction: ${instruction}
# Source: ${source}
# Model-generated output: ${model_generated}
# Identified errors along with their explanations in the model-generated output are as follows: 
# ${identified_errors}

# However, some of the identified errors might contain some problems. Some problems types includes:
# - The pointed error location does not contain any error
# - The pointed error location contains an error, but the correction is actually same as the model-generated output.
# - The pointed error location contains an error, but the correction does not fix the error.

# For each identified error, considering the instruction and source text, answer whether the identified error contains any problem mentioned above. 
# Output choices are as follows:
# - true: for the identified error contains any problem mentioned above
# - false: for the identified error does not contain any problem mentioned above. That is, the identified error is indeed an error; the explanation is correct and makes sense; the correction does fix the error 
# Output format:
# {"error_1": true/false, ...}
# """

check_template_for_no_errors = """
Instruction: ${instruction}
Source: ${source}
Model-generated output: ${model_generated}
An annotator has been asked to check if there is any error in the model-generated output. The annotator has answered that there is no error in the model-generated output.
However, the annotator might miss some errors. Please check if there is any error in the model-generated output.
Output format:
T/F (T: true for there is an error in the model-generated output; F: false for there is no error in the model-generated output)
"""

def get_check_prompts(
    item: dict
):
    check_prompts = []
    for cand in item['candidates']:
        if not isinstance(cand['responses'][1], dict):
            check_prompts.append(None)
            continue
        if len(cand['responses'][1]['errors']) == 0:
            check_prompt = Template(check_template_for_no_errors).substitute(
                instruction=item['instruction'],
                source=item['input'],
                model_generated=cand['text'],
            )
        else:
            # shuffle errors
            shuffled_errors = list(cand['responses'][1]['errors'].values())
            random.shuffle(shuffled_errors)
            cand['responses'][1]["errors"] = {f"error_{i+1}": v for i, v in enumerate(shuffled_errors)}
            check_prompt = Template(check_template).substitute(
                instruction=item['instruction'],
                source=item['input'],
                reference=item.get('output') or item.get("refs")[0],
                model_generated=cand['text'],
                identified_errors=json.dumps(cand['responses'][1], ensure_ascii=False, indent=0),   
            )
        check_prompt = check_prompt.strip(" \n\t")
        check_prompts.append(check_prompt)
    
    return check_prompts

def main(
    input_file: str,
    output_file: str,
    seed: int = 42,
    model_name: str = "ChatGPT",
):
    random.seed(seed)
    with open(input_file, "r") as f:
        data = json.load(f)[:10]
    check_prompts = list(map(get_check_prompts, data))
    full_check_prompts = [item for sublist in check_prompts for item in sublist]
    check_prompts = [x for x in full_check_prompts if x is not None]
    print(len(check_prompts))
    print(len(full_check_prompts))
    chatmls = [[{"role":"system","content":"You are an AI assistant that helps people find information."},
            {"role":"user","content": prompt}] for prompt in check_prompts]
    chatml_prompts = [_chatml_to_prompt(chatml) for chatml in chatmls]

    decoding_kwargs = {
        # "max_tokens": 1024,
        "temperature": 0,
        "top_p": 1.0,
        "timeout": 30,
        "request_timeout": 30
    }
    results = openai_completions(chatml_prompts, model_name=model_name, **decoding_kwargs)
    logging.info("Total price: {:.4f}$".format(sum(results['price_per_example'])))
    completions = results['completions']
    
    def loads_json(json_str: str):
        try:
            return json.loads(json_str)
        except:
            return json_str
    idx = 0
    full_idx = 0
    for item in data:
        for cand in item['candidates']:
            if full_check_prompts[idx] is None:
                cand['check_prompt'] = None
                cand['check_response'] = None
            else:
                cand['check_prompt'] = check_prompts[idx]
                cand['check_response'] = loads_json(completions[idx])
                idx += 1
            full_idx += 1
    with open(output_file, "w") as f:
        json.dump(data, f, indent=4, ensure_ascii=False)
    logging.info(f"Saved to {output_file}")

    # # counter the distribution of each QA
    # qa_answers = {f"Q{i+1}" : [] for i in range(11)}
    # for item in data:
    #     for cand in item['candidates']:
    #         if isinstance(cand['check_response'], dict):
    #             for q in cand['check_response']:
    #                 if re.match(r"Q\d+", q):
    #                     qa_answers[q].append(cand['check_response'][q])
    #                 elif re.match(r"error_\d+", q):
    #                     for q2 in cand['check_response'][q]:
    #                         if re.match(r"Q\d+", q2):
    #                             qa_answers[q2].append(cand['check_response'][q][q2])
    #                         else:
    #                             raise ValueError(f"Invalid question {q2}")
    #                 else:
    #                     raise ValueError(f"Invalid question {q}")
    # for q in qa_answers:
    #     if q == "Q1":
    #         logging.info(f"{q}: {Counter([type(item) for item in qa_answers[q]])}")
    #     else:
    #         logging.info(f"{q}: {Counter([str(item) for item in qa_answers[q]])}")

    # # filter out the canidates based on the answers
    # filtered_data = copy.deepcopy(data)
    # for item in filtered_data:
    #     for i, cand in enumerate(item['candidates']):
    #         flag = True
    #         keep_cand_idxs = []
    #         if not isinstance(cand['check_response'], dict):
    #             continue
    #         for key in cand['check_response']:
    #             if re.match(r'error_\d+', key):
    #                 for sub_key in cand['check_response'][key]:
    #                     if sub_key == "Q1":
    #                         pass
    #                     elif sub_key == "Q2":
    #                         if cand['check_response'][key][sub_key] == "No":
    #                             flag = False
    #                             break
    #                     elif sub_key == "Q3":
    #                         if cand['check_response'][key][sub_key] == "No":
    #                             flag = False
    #                             break
    #                     elif sub_key == "Q4":
    #                         if isinstance(cand['check_response'][key][sub_key], int):
    #                             cand['responses'][-1]['errors'][key]['reduction_score'] = cand['check_response'][key][sub_key]
    #                     elif sub_key == "Q5":
    #                         if cand['check_response'][key][sub_key] == "No":
    #                             flag = False
    #                             break
    #                     elif sub_key == "Q5":
    #                         if cand['check_response'][key][sub_key] == "No":
    #                             flag = False
    #                             break
    #                     elif sub_key == "Q6":
    #                         if cand['check_response'][key][sub_key] == "Yes":
    #                             flag = False
    #                             break
    #                     else:
    #                         raise ValueError(f"Invalid question {sub_key}")
    #         if flag:
    #             keep_cand_idxs.append(i)
    #     item['candidates'] = [item['candidates'][i] for i in keep_cand_idxs]
    # # count the number of removed candidates
    # num_removed = 0
    # ori_total_num_cands = 0
    # for filtered_item, item in zip(filtered_data, data):
    #     num_removed += len(item['candidates']) - len(filtered_item['candidates'])
    #     ori_total_num_cands += len(item['candidates'])
    # logging.info(f"Removed {num_removed} candidates from {ori_total_num_cands} candidates")
    # # filter out the items with no candidates
    # filtered_data = [item for item in filtered_data if len(item['candidates']) > 0]
    # logging.info(f"Filtered {len(data) - len(filtered_data)} items")
    # # save the filtered data
    # with open(output_file.replace(".json", "_filtered.json"), "w") as f:
    #     json.dump(filtered_data, f, indent=4, ensure_ascii=False)
    # logging.info(f"Saved to {output_file.replace('.json', '_filtered.json')}")
    
if __name__ == "__main__":
    fire.Fire(main)
tigerscore/eval_scripts/check_responses.py
TIGER-AI-Lab/TIGERScore
import fire
import sys
import os
import json
import scipy
import numpy as np
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from common.evaluation import overall_eval
from typing import List 
import prettytable as pt
from collections import defaultdict
from mt_metrics_eval.stats import Correlation

class MyCorrelation(Correlation):
    def __init__(self, num_sys:int, gold_scores:List[int], metric_scores:List[int]):
        # remove nan in metrics scores
        none_metric_scores_idxs = [idx for idx, x in enumerate(metric_scores) if x is None]
        # print("Remove {} nan scores from {} scores".format(
        #     len(none_metric_scores_idxs),
        #     len(metric_scores)
        # ))
        gold_scores = gold_scores.copy()
        # set gold scores to None if metric scores are None
        for idx in none_metric_scores_idxs[::-1]:
            gold_scores[idx] = None 
        super().__init__(num_sys, gold_scores, metric_scores)
    
def get_ranks_from_scores(scores):
    """
    Args:
        scores: ndarray of shape (n, c) or (c) where n is the number of samples, c is the number of candidates
        Treat same as higher one
        
    Returns:
        ranks: ndarray of shape (n, c) or (c) where n is the number of samples, c is the number of candidates
    """
    if isinstance(scores, list):
        scores = np.array(scores)
    orig_shape = scores.shape
    if len(scores.shape) == 1:
        scores = scores.reshape(1, -1)
    bz, c = scores.shape
    ranks = np.zeros((bz, c), dtype=np.int32)
    for i in range(bz):
        sorted_scores_i = list(sorted(list(scores[i]), reverse=True))
        for j in range(c):
            ranks[i, j] = sorted_scores_i.index(scores[i, j]) + 1
    
    ranks = ranks.reshape(orig_shape)
    return ranks

def cor_spearman(hypo_ranks, ref_ranks):
    """
    Args:
        hypo_ranks: ndarray of shape (n, c) where n is the number of samples, c is the number of candidates
        ref_ranks: ndarray of shape (n, c) where n is the number of samples, c is the number of candidates
    returns:
        cor: float, the mean of the diagonal elements of the spearman correlation matrix
    """
    if isinstance(hypo_ranks, list):
        hypo_ranks = np.array(hypo_ranks)
    if isinstance(ref_ranks, list):
        ref_ranks = np.array(ref_ranks)
    assert hypo_ranks.shape == ref_ranks.shape
    bz, c = hypo_ranks.shape
    hypo_ranks = hypo_ranks.reshape(bz, c).T
    ref_ranks = ref_ranks.reshape(bz, c).T
    cor = 0
    for i in range(c):
        cor += scipy.stats.spearmanr(hypo_ranks[i], ref_ranks[i]).correlation
    cor /= c
    return cor

def main(
    input_file: str,
    output_file: str,
    human_score_names: List[str],
    metrics: List[str],
    num_workers: int = 1,
    overwrite: bool = False,
    print_results: bool = False,
    average_by: str = "none",
    as_rank: bool = False,
):
    if isinstance(metrics, str):
        metrics = metrics.split(" ")
    else:
        metrics = list(metrics)
    if isinstance(human_score_names, str):
        human_score_names = [human_score_names]
    else:
        human_score_names = list(human_score_names)
    if not os.path.exists(output_file):
        with open(input_file) as f:
            data = json.load(f)
        with open(output_file, "w") as f:
            json.dump(data, f, indent=4, ensure_ascii=False)
            print(f"Saved a inital copy to {output_file}")
    else:
        with open(output_file) as f:
            data = json.load(f)
        with open(input_file) as f:
            data_input = json.load(f)
        if len(data) != len(data_input):
            print("Warning: data and data_input have different length")
            with open(output_file, "w") as f:
                json.dump(data_input, f, indent=4, ensure_ascii=False)
                print(f"Saved a inital copy to {output_file}")
            data = data_input

    evaled_metrics = []
    for metric in metrics:
        if any("my_"+metric in score_name for score_name in data[0]['candidates'][0]['scores'].keys()):
            # ######## Temp Code remember delete #########
            # if "_src_hypo" in metric:
            #     continue
            evaled_metrics.append(metric)
    if not overwrite:
        to_eval_metrics = [metric for metric in metrics if metric not in evaled_metrics]
    else:
        to_eval_metrics = metrics
    print(metrics)
    print("To eval metrics: {}".format(to_eval_metrics))
    if len(to_eval_metrics) > 0:
        sources = [d["input"] for d in data]
        references = [d.get("output") or d['refs'][0] for d in data]
        hypotheses = [[cand['text'] for cand in d['candidates']] for d in data]
        scores = overall_eval(hypotheses, references, to_eval_metrics, sources=sources, num_workers=num_workers)
        for metric, _scores in scores.items():
            for i, item in enumerate(data):
                for j, cand in enumerate(item['candidates']):
                    cand['scores']["my_"+metric] = _scores[i][j]
        with open(output_file, "w") as f:
            json.dump(data, f, indent=4, ensure_ascii=False)
            print(f"Saved to {output_file}")
    
    if print_results:
        # compute correlations
        human_score_corr_dict = {human_score_name: defaultdict(dict) for human_score_name in human_score_names}
        all_scores_names = list(data[0]['candidates'][0]['scores'].keys())
        other_scores_names = [score_name for score_name in all_scores_names if score_name not in human_score_names]
        # remove logprobs
        if "logprobs" in other_scores_names:
            other_scores_names.remove("logprobs")
        # remove redundant bart_score_cnn_src_hypo and bart_score_cnn_ref_hypo
        bartscore_redundant_scores_names = [x for x in other_scores_names if x.startswith("bart_score_cnn_src_hypo") or x.startswith("bart_score_cnn_ref_hypo")]
        for score_name in bartscore_redundant_scores_names:
            other_scores_names.remove(score_name)
        for h_name in human_score_names:
            num_sys = len(data[0]['candidates'])
            h_scores = [[cand['scores'][h_name] for cand in item['candidates']] for item in data ]
            
            for o_name in other_scores_names:
                o_scores = [[cand['scores'][o_name] for cand in item['candidates']] for item in data ]
                if as_rank:
                    # transform scores to ranks before computing correlation
                    o_scores = get_ranks_from_scores(o_scores)
                    o_scores = (- np.array(o_scores)).tolist() # higher is better
                h_scores_flat = []
                h_scores_T = [x for x in zip(*h_scores)]
                for sublist in h_scores_T:
                    h_scores_flat.extend(sublist)
                # h_scores = np.array(h_scores).T.reshape(-1).tolist()
                o_scores_flat = []
                o_scores_T = [x for x in zip(*o_scores)]
                for sublist in o_scores_T:
                    o_scores_flat.extend(sublist)
                # o_scores = np.array(o_scores).T.reshape(-1).tolist()
                corr = MyCorrelation(1,h_scores_flat, o_scores_flat)
                human_score_corr_dict[h_name][o_name]['Pearson'] = [round(x, 4) for x in corr.Pearson(average_by=average_by)]
                human_score_corr_dict[h_name][o_name]['Spearman'] = [round(x, 4) for x in corr.Spearman(average_by=average_by)]
                human_score_corr_dict[h_name][o_name]['Kendall'] = [round(x, 4) for x in corr.Kendall(average_by=average_by)]

        # print table
        table = pt.PrettyTable()
        table.field_names = ["Human Score", "Metric", "Pearson", "Spearman", "Kendall"]
        table.align["Human Score"] = "l"
        table.align["Metric"] = "l"
        table.align["Pearson"] = "l"
        table.align["Spearman"] = "l"
        table.align["Kendall"] = "l"
        # add data
        for h_name in human_score_names:
            other_scores_pearson = [human_score_corr_dict[h_name][o_name]['Pearson'][0] for o_name in other_scores_names]
            sorted_other_scores_names = [x for _, x in sorted(zip(other_scores_pearson, other_scores_names), reverse=True)]
            for o_name in sorted_other_scores_names:
                table.add_row([
                    h_name,
                    o_name,
                    human_score_corr_dict[h_name][o_name]['Pearson'],
                    human_score_corr_dict[h_name][o_name]['Spearman'],
                    human_score_corr_dict[h_name][o_name]['Kendall'],
                ])
            if h_name != human_score_names[-1]:
                table.add_row(["-", "-", "-", "-", "-"])
        print("File: {}".format(output_file))
        print("Correlations (Sorted by Pearson):")
        print(table)

if __name__ == "__main__":
    fire.Fire(main)

tigerscore/eval_scripts/eval_baseline.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_wmt.py
"""
import json
import random
import logging
import sys
import fire
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json
logging.basicConfig(level=logging.warning)

# # params
# task='translation'
# xgptscore_mode="wmt_mqm"
# version_key=f"{xgptscore_mode}.distill_new_wmt_mqm"
# human_score_names=["mqm", "da"]
# model_name="ChatGPT"
# overwrite=True
# max_size=None # set to None to use all examples
# if isinstance(max_size, int) and max_size > 0:
#     version_key = f"{version_key}_{max_size}"
# # load data
#     split="train"
#     data_dir="../../data"
#     input_file=Path(f"{data_dir}/wmt/ru-en/{split}_data.json")

def main(
    task: str,
    xgptscore_mode: str,
    model_name: str,
    input_file: str,
    version_key: str = None,
    overwrite: bool = False,
    max_size: int = None,
    seed: int = 42,
    shuffle: bool = False,
    source_max_length: int = None,
    ref_max_length: int = None,
    hypo_max_length: int = None,
):
    
    logging.warning("Loading from {}".format(input_file))
    with open(input_file, "r") as f:
        items = json.load(f)
    if shuffle:
        random.seed(seed)
        random.shuffle(items)
    suffix = f".{xgptscore_mode}.{model_name}"
    if version_key:
        suffix += f".{version_key}"
    if isinstance(max_size, int) and max_size > 0:
        items = items[:max_size]
        suffix += f".{max_size}"
    output_file = Path(input_file).with_suffix(f"{suffix}.json")

    xgptitems = []
    for item in items:
        for cand in item['candidates']:
            xgptitems.append(XPGTItem(
                task=task,
                instruction=item['instruction'],
                input=item['input'],
                ref_output=item['refs'] if 'refs' in item else item['output'],
                hypo_output=cand['text']
            ))

    if not output_file.exists() or overwrite:
        logging.warning("Running xgptscore")
        # run xgptscore
        xgptscore_params = {
            "max_lengths": {
                "input": source_max_length,
                "hypo_output": hypo_max_length,
                "ref_output": ref_max_length,
            },
        }
        result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name, **xgptscore_params)
        idx = 0
        for item in items:
            for cand in item['candidates']:
                cand['responses'] = result['round_completions'][idx]
                cand['messages_records'] = result['messages_records'][idx]
                if 'scores' not in cand:
                    cand['scores'] = {}
                try:
                    cand['scores']['xgptscore'] = get_xgptscore_from_json(cand['responses'][-1])
                except:
                    pass
                idx += 1
        with open(output_file, "w") as f:
            json.dump(items, f, indent=4, ensure_ascii=False)
            logging.warning("Saved to {}".format(output_file))
    else:
        logging.warning("Found existing {}".format(output_file))
        logging.warning("Skipping xgptscore")

if __name__ == "__main__":
    logging.basicConfig(level=logging.warning)
    fire.Fire(main)
tigerscore/eval_scripts/generate_distill_data.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python generate_distill_d2t.py
"""
import json
import random
import logging
import sys
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json_per_aspect
from xgptscore.constants import EVAL_ASPECTS
logging.basicConfig(level=logging.INFO)

# params
task='data2text'
xgptscore_mode="d2t"
version_key=f"{xgptscore_mode}.distill_new"
human_score_names=[]
model_name="ChatGPT"
overwrite=False
max_size=None # set to None to use all examples
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
split="train"
data_dir="../../data"
input_file=Path(f"{data_dir}/d2t/train_data_new.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")
logging.info("Loading from {}".format(input_file))
with open(input_file, "r") as f:
    items = json.load(f)
if isinstance(max_size, int) and max_size > 0:
    items = items[:max_size]

xgptitems = []
for item in items:
    for cand in item['candidates']:
        xgptitems.append(XPGTItem(
            task=task,
            instruction=item['instruction'],
            input=item['input'],
            ref_output=item['output'],
            hypo_output=cand['text']
        ))

if not output_file.exists() or overwrite:
    logging.info("Running xgptscore")
    # run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name,num_workers=5)
    idx = 0
    aspects = EVAL_ASPECTS[task].keys()
    score_dict = {"xgptscore_"+aspect: 0 for aspect in aspects}
    for item in items:
        for cand in item['candidates']:      
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            xgptscore_ans = get_xgptscore_from_json_per_aspect(cand['responses'][-1])
            if xgptscore_ans is None:
                logging.info(f"XGPTScore failed for {cand['text']}")
                # cand['scores']['xgptscore'] = None
            else:
                cand['scores'].update(score_dict)
                cand['scores'].update(xgptscore_ans) 
            idx += 1
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)

tigerscore/eval_scripts/generate_distill_data_d2t.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python generate_distill_d2t.py
"""
import json
import random
import logging
import sys
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json_per_aspect
from xgptscore.constants import EVAL_ASPECTS
logging.basicConfig(level=logging.INFO)

# params
task='instruction-following'
xgptscore_mode="instruction_following"
version_key=f"{xgptscore_mode}_new.distill"
human_score_names=[]
model_name="ChatGPT"
overwrite=False
max_size=None # set to None to use all examples
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
split="train"
data_dir="../../data"
input_file=Path(f"{data_dir}/ins/train_data_new.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")
logging.info("Loading from {}".format(input_file))
with open(input_file, "r") as f:
    items = json.load(f)
if isinstance(max_size, int) and max_size > 0:
    items = items[:max_size]

xgptitems = []
for item in items:
    for cand in item['candidates']:
        xgptitems.append(XPGTItem(
            task=task,
            instruction=item['instruction'],
            input=item['input'],
            ref_output=item['output'],
            hypo_output=cand['text']
        ))

if not output_file.exists() or overwrite:
    logging.info("Running xgptscore")
    # run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name,num_workers=5)
    idx = 0
    aspects = EVAL_ASPECTS[task].keys()
    score_dict = {"xgptscore_"+aspect: 0 for aspect in aspects}
    for item in items:
        for cand in item['candidates']:      
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            xgptscore_ans = get_xgptscore_from_json_per_aspect(cand['responses'][-1])
            if xgptscore_ans is None:
                logging.info(f"XGPTScore failed for {cand['text']}")
                # cand['scores']['xgptscore'] = None
            else:
                cand['scores'].update(score_dict)
                cand['scores'].update(xgptscore_ans) 
            idx += 1
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)

tigerscore/eval_scripts/generate_distill_data_ins.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python generate_distill_d2t.py
"""
import json
import random
import logging
import sys
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json_per_aspect
from xgptscore.constants import EVAL_ASPECTS
logging.basicConfig(level=logging.INFO)

# params
task='long-form QA'
xgptscore_mode="longform_qa"
version_key=f"{xgptscore_mode}.distill"
human_score_names=[]
model_name="ChatGPT"
overwrite=False
max_size=None # set to None to use all examples
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
split="train"
data_dir="../../data"
input_file=Path(f"{data_dir}/lfqa/train_data_new.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")
logging.info("Loading from {}".format(input_file))
with open(input_file, "r") as f:
    items = json.load(f)
if isinstance(max_size, int) and max_size > 0:
    items = items[:max_size]

xgptitems = []
for item in items:
    for cand in item['candidates']:
        xgptitems.append(XPGTItem(
            task=task,
            instruction=item['instruction'],
            input=item['input'],
            ref_output=item['output'],
            hypo_output=cand['text']
        ))

if not output_file.exists() or overwrite:
    logging.info("Running xgptscore")
    # run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name,num_workers=5)
    idx = 0
    aspects = EVAL_ASPECTS[task].keys()
    score_dict = {"xgptscore_"+aspect: 0 for aspect in aspects}
    for item in items:
        for cand in item['candidates']:      
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            xgptscore_ans = get_xgptscore_from_json_per_aspect(cand['responses'][-1])
            if xgptscore_ans is None:
                logging.info(f"XGPTScore failed for {cand['text']}")
                # cand['scores']['xgptscore'] = None
            else:
                cand['scores'].update(score_dict)
                cand['scores'].update(xgptscore_ans) 
            idx += 1
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)

tigerscore/eval_scripts/generate_distill_data_lfqa.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python generate_distill_d2t.py
"""
import json
import random
import logging
import sys
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json
from xgptscore.constants import EVAL_ASPECTS
logging.basicConfig(level=logging.WARNING)

# params
task='mathQA'
xgptscore_mode="mathqa"
version_key=f"{xgptscore_mode}.distill"
human_score_names=[]
model_name="ChatGPT"
overwrite=True
max_size=200 # set to None to use all examples
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{model_name}_{version_key}_{max_size}"

# load data
split="train"
data_dir="../../data_bak"
input_file=Path(f"{data_dir}/mathqa/gsm8k_test_output_prepared.ChatGPT_score.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")
logging.warning("Loading from {}".format(input_file))
with open(input_file, "r") as f:
    items = json.load(f)
if isinstance(max_size, int) and max_size > 0:
    items = items[:max_size]

xgptitems = []
for item in items:
    for cand in item['candidates']:
        xgptitems.append(XPGTItem(
            task=task,
            instruction=item['instruction'],
            input=item['input'],
            ref_output=item['output'],
            hypo_output=cand['text']
        ))

if not output_file.exists() or overwrite:
    logging.warning("Running xgptscore")
    # run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name,num_workers=5)
    idx = 0
    for item in items:
        for cand in item['candidates']:      
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            cand['scores']['xgptscore'] = get_xgptscore_from_json(cand['responses'][-1])
            idx += 1
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.warning("Saved to {}".format(output_file))
else:
    logging.warning("Loading from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)

tigerscore/eval_scripts/generate_distill_data_mathqa.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_wmt.py
"""
import json
import random
import logging
import sys
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json_per_aspect
from xgptscore.constants import EVAL_ASPECTS
logging.basicConfig(level=logging.INFO)

# params
task='summarization'
xgptscore_mode="align_score"
version_key=f"{xgptscore_mode}.distill"
human_score_names=[]
model_name="ChatGPT"
overwrite=True
max_size=None # set to None to use all examples
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
split="train"
data_dir="../../data"
input_file=Path(f"{data_dir}/sum/train_data_new.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")
logging.info("Loading from {}".format(input_file))
with open(input_file, "r") as f:
    items = json.load(f)
if isinstance(max_size, int) and max_size > 0:
    items = items[:max_size]

xgptitems = []
for item in items:
    for cand in item['candidates']:
        xgptitems.append(XPGTItem(
            task=task,
            instruction=item['instruction'],
            input=item['input'],
            ref_output=item['output'],
            hypo_output=cand['text']
        ))

if not output_file.exists() or overwrite:
    logging.info("Running xgptscore")
    # run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name)
    idx = 0
    aspects = EVAL_ASPECTS[task].keys()
    score_dict = {"xgptscore_"+aspect: 0 for aspect in aspects}
    for item in items:
        for cand in item['candidates']:      
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            xgptscore_ans = get_xgptscore_from_json_per_aspect(cand['responses'][-1])
            if xgptscore_ans is None:
                logging.info(f"XGPTScore failed for {cand['text']}")
                # cand['scores']['xgptscore'] = None
            else:
                cand['scores'].update(score_dict)
                cand['scores'].update(xgptscore_ans) 
            idx += 1
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)

tigerscore/eval_scripts/generate_distill_data_sum.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_wmt.py
"""
import json
import random
import logging
import sys
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json
logging.basicConfig(level=logging.INFO)

# params
task='translation'
xgptscore_mode="wmt_mqm"
version_key=f"{xgptscore_mode}.distill_new_wmt_mqm"
human_score_names=["mqm", "da"]
model_name="ChatGPT"
overwrite=True
max_size=None # set to None to use all examples
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
split="train"
data_dir="../../data"
input_file=Path(f"{data_dir}/wmt/ru-en/{split}_data.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")
logging.info("Loading from {}".format(input_file))
with open(input_file, "r") as f:
    items = json.load(f)
random.seed(42)
random.shuffle(items)
if isinstance(max_size, int) and max_size > 0:
    items = items[:max_size]

xgptitems = []
for item in items:
    for cand in item['candidates']:
        xgptitems.append(XPGTItem(
            task=task,
            instruction=item['instruction'],
            input=item['input'],
            ref_output=item['refs'],
            hypo_output=cand['text']
        ))

if not output_file.exists() or overwrite:
    logging.info("Running xgptscore")
    # run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name)
    idx = 0
    for item in items:
        for cand in item['candidates']:
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            cand['scores']['xgptscore'] = get_xgptscore_from_json(cand['responses'][-1])
            idx += 1
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)

# Compute correlation
if human_score_names is not None:
    logging.info("Computing correlation with human score {}".format(human_score_names))
    human_scores = []
    for item in items:
        for cand in item['candidates']:
            for h_name in human_score_names:
                if h_name in cand['scores']:
                    human_scores.append(cand['scores'][h_name])
                    break

    metrics = ["xgptscore"]

    Pearson_corr = {}
    Spearman_corr = {}
    Kendall_corr = {}
    for metric in metrics:
        metric_scores = [[cand['scores'][metric] for cand in item['candidates']] for item in items]
        metric_scores = list(chain(*metric_scores)) # transpose and flatten
        metric_corr = MyCorrelation(1, human_scores, metric_scores)
        Pearson_corr[metric] = metric_corr.Pearson()
        Spearman_corr[metric] = metric_corr.Spearman()
        Kendall_corr[metric] = metric_corr.Kendall()

    # sort Corr
    Pearson_corr = {k: v for k, v in sorted(Pearson_corr.items(), key=lambda item: item[1][0], reverse=True)}
    Spearman_corr = {k: v for k, v in sorted(Spearman_corr.items(), key=lambda item: item[1][0], reverse=True)}
    Kendall_corr = {k: v for k, v in sorted(Kendall_corr.items(), key=lambda item: item[1][0], reverse=True)}
    Corr_record = {
        "Pearson": Pearson_corr,
        "Spearman": Spearman_corr,
        "Kendall": Kendall_corr,
    }
    # Save correlation results
    corr_results_file = Path("./eval_results/") / output_file.relative_to(data_dir).with_suffix(".corr.json")
    corr_results_file.parent.mkdir(parents=True, exist_ok=True)
    with open(corr_results_file, "w") as f:
        json.dump(Corr_record, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(corr_results_file))
else:
    logging.info("No human score provided, skip correlation computation")
    


tigerscore/eval_scripts/generate_distill_data_wmt.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_wmt.py
"""
import json
import random
import logging
import sys
import fire
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem
logging.basicConfig(level=logging.warning)

def main(
    task: str,
    input_file: str,
    output_file: str = None,
    xgptscore_mode: str = "kb_txt",
    model_name: str = "gpt-4",
    version_key: str = "default",
    overwrite: bool = False,
    max_size: int = None,
    seed: int = 42,
    shuffle_file: bool = False,
    source_max_length: int = None,
    ref_max_length: int = None,
    hypo_max_length: int = None,
):
    logging.warning("Params: \n{}".format(json.dumps(locals(), indent=4)))
    # params
    if isinstance(max_size, int) and max_size > 0:
        version_key = f"{version_key}_{max_size}"
    # load data
    input_file=Path(input_file)
    if not output_file:
        output_file = input_file.with_suffix(f".{xgptscore_mode}.{version_key}.json")
    else:
        output_file = Path(output_file)
    with open(input_file, "r") as f:
        items = json.load(f)
        logging.warning("Loaded {} items from {}".format(len(items), input_file))
    logging.warning("Preparing writing to {}...".format(output_file))
    
    random.seed(seed); logging.warning("Set seed to {}".format(seed))
    if shuffle_file:
        random.shuffle(items); logging.warning("Shuffled {} items".format(len(items)))
    if isinstance(max_size, int) and max_size > 0:
        items = items[:max_size]; logging.warning("Truncated to {} items".format(len(items)))
    elif isinstance(max_size, float) and max_size > 0 and max_size < 1:
        items = random.sample(items, int(len(items) * max_size)); logging.warning("Sampled to {} items".format(len(items)))
    

    xgptitems = []
    for item in items:
        xgptitems.append(XPGTItem(
            task=task,
            instruction=item['instruction'],
            input=item['input'],
            ref_output=item['output'] if "output" in item else item['refs'],
            hypo_output=None,
        ))
        if "candidates" in item:
            del item["candidates"]

    if not output_file.exists() or overwrite:
        logging.warning("Running xgptscore")
        # run xgptscore
        xgptscore_params = {
            "max_lengths": {
                "input": source_max_length,
                "hypo_output": hypo_max_length,
                "ref_output": ref_max_length,
            },
        }
        result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name, **xgptscore_params)
        for i, item in enumerate(items):
            item['responses'] = result['round_completions'][i]
            item['messages_records'] = result['messages_records'][i]
        with open(output_file, "w") as f:
            json.dump(items, f, indent=4, ensure_ascii=False)
            logging.warning("Saved to {}".format(output_file))
    else:
        logging.warning("Loading from {}".format(output_file))
        with open(output_file, "r") as f:
            items = json.load(f)

if __name__ == "__main__":
    fire.Fire(main)
tigerscore/eval_scripts/generate_synthesis_distill_data.py
TIGER-AI-Lab/TIGERScore
import sys
import fire
import json
import logging
import regex as re
import copy
import random
sys.path.append("..")
from xgptscore.openai_utils import openai_completions, _chatml_to_prompt
from typing import List, Dict
from string import Template
from collections import Counter, defaultdict
logging.basicConfig(level=logging.WARNING)

rank_template = """
4 different models are asked to follow a given instruction to generate an answer based on a given source input.
The instruction is: ${instruction}
The source input is: ${source}
The generated output of model 1 is: ${model1_generated}
The generated output of model 2 is: ${model2_generated}
The generated output of model 3 is: ${model3_generated}
The generated output of model 4 is: ${model4_generated}
The reference output is: ${reference}

Now Please rank the 4 model's outputs from best to worst. 
Please first output the rank results in the following format:
[best] [second best] [third best] [worst] (e.g. 1 2 3 4)
Then give your brief comments on why you rank the outputs in this way.
"""


def get_rank_prompts(
    item: dict
):
    random.shuffle(item['candidates'])
    rank_prompt = Template(rank_template).substitute(
        instruction=item['instruction'],
        source=item['input'],
        model1_generated=item['candidates'][0]['text'],
        model2_generated=item['candidates'][1]['text'],
        model3_generated=item['candidates'][2]['text'],
        model4_generated=item['candidates'][3]['text'],
        reference=item.get('output') or item.get("refs")[0],
    )
    return rank_prompt 

def main(
    input_file: str,
    output_file: str,
    seed: int = 42,
    model_name: str = "ChatGPT",
):
    random.seed(seed)
    with open(input_file, "r") as f:
        data = json.load(f)
    
    rank_prompts = list(map(get_rank_prompts, data))
    chatmls = [[{"role":"system","content":"You are an helpful AI assistant to help user find information."},
            {"role":"user","content": prompt}] for prompt in rank_prompts]
    chatml_prompts = [_chatml_to_prompt(chatml) for chatml in chatmls]

    decoding_kwargs = {
        # "max_tokens": 1024,
        "temperature": 0,
        "top_p": 1.0,
        "timeout": 30,
        "request_timeout": 30
    }
    results = openai_completions(chatml_prompts, model_name=model_name, **decoding_kwargs)
    logging.warning("Total price: {:.4f}$".format(sum(results['price_per_example'])))
    completions = results['completions']
    
    best_model_idxs = []
    model_ranks = defaultdict(list)
    for i, item in enumerate(data):
        item['rank_prompt'] = rank_prompts[i]
        item['rank_response'] = completions[i]
        try:
            first_digit_idx = re.search(r"\d", item['rank_response']).start()
            item['ranks'] = re.search(r"(\d)[\n ](\d)[\n ](\d)[\n ](\d)", item['rank_response'])
            if not item['ranks']:
                item['ranks'] = re.search("\[best\] (\d) \[second best\] (\d) \[third best\] (\d) \[worst\] (\d)", item['rank_response'])
            if not item['ranks']:
                item['ranks'] = re.search("\[best\] Model (\d)[\n ]\[second best\] Model (\d)[\n ]\[third best\] Model (\d)[\n ]\[worst\] Model (\d)", item['rank_response'])
            # item['ranks'] = item['rank_response'][first_digit_idx:item['rank_response'].index("\n")].split(" ")
            item['ranks'] = [int(rank) for rank in item['ranks'].groups()]
        except:
            print(item['ranks'])
        for j, cand in enumerate(item['candidates']):
            cand['scores']['gpt_rank_{}'.format(model_name)] = - item['ranks'][j]
            model_ranks[cand['source']].append(item['ranks'][j])
        best_model_idxs.append(item['ranks'][0])
                
        
    print(Counter(best_model_idxs))
    for model, ranks in model_ranks.items():
        c = Counter(ranks)
        print(model, sorted(c.items(), key=lambda x: x[0]))
    with open(output_file, "w") as f:
        json.dump(data, f, indent=4, ensure_ascii=False)
    logging.warning(f"Saved to {output_file}")

if __name__ == "__main__":
    fire.Fire(main)
tigerscore/eval_scripts/lfqa_gpt_rate.py
TIGER-AI-Lab/TIGERScore
import sys
import fire
import json
import logging
import regex as re
import copy
import random
sys.path.append("..")
from xgptscore.openai_utils import openai_completions, _chatml_to_prompt
from typing import List, Dict
from string import Template
from collections import Counter, defaultdict
logging.basicConfig(level=logging.WARNING)

template = """
${instruction}
${source}

A correct output is: 
${reference}

A model generated output is:
${model1_generated}

Now please evaluate the errors in the model-generated outputs
For each error associated with problem understanding, problem formulation,  computing accuracy, and solution interpretation, reduce 1 or 2 score. 
Finally give me a total reductions of score as the evaluation of this model-generated output starting with "Total Score Reduction: ".
"""


def get_prompts(
    item: dict
):
    prompts = []
    random.shuffle(item['candidates'])
    for cand in item['candidates']:
        prompt = Template(template).substitute(
            instruction=item['instruction'].strip("\n "),
            source=item['input'].strip("\n "),
            reference=(item.get('output') or item.get("refs")[0]).strip("\n "),
            model1_generated=cand['text'].strip("\n "),
        )
        prompts.append(prompt)
    return prompts

def main(
    input_file: str,
    output_file: str,
    seed: int = 42,
    model_name: str = "ChatGPT",
):
    random.seed(seed)
    with open(input_file, "r") as f:
        data = json.load(f)
    
    prompts = list(map(get_prompts, data))
    flatten_prompts = [prompt for prompts_ in prompts for prompt in prompts_]
    chatmls = [[{"role":"system","content":"You are an helpful AI assistant to help user find information."},
            {"role":"user","content": prompt}] for prompt in flatten_prompts]
    chatml_prompts = [_chatml_to_prompt(chatml) for chatml in chatmls]

    decoding_kwargs = {
        # "max_tokens": 1024,
        "temperature": 0,
        "top_p": 1.0,
        "timeout": 30,
        "request_timeout": 30
    }
    results = openai_completions(chatml_prompts, model_name=model_name, **decoding_kwargs)
    logging.warning("Total price: {:.4f}$".format(sum(results['price_per_example'])))
    completions = results['completions']
    
    idx = 0
    for i, item in enumerate(data):
        for j, cand in enumerate(item['candidates']):
            total_score_reduction = re.search("Total Score Reduction: (\d+)", completions[idx])
            if not total_score_reduction:
                total_score_reduction = re.search("Total Score Reduction: -(\d+)", completions[idx])
            if not total_score_reduction:
                total_score_reduction = re.search("Total Score Reduction is (\d+)", completions[idx])
            if not total_score_reduction:
                total_score_reduction = re.search("Total Score Reduction is -(\d+)", completions[idx])
            if total_score_reduction:
                cand['scores']['gpt_score_reduction'] = - abs(int(total_score_reduction.groups()[0]))
            else:
                pass
                cand['scores']['gpt_score_reduction'] = 0
            cand['gpt_score_output'] = completions[idx]
            idx += 1
    with open(output_file, "w") as f:
        json.dump(data, f, indent=4, ensure_ascii=False)
    logging.warning(f"Saved to {output_file}")

if __name__ == "__main__":
    fire.Fire(main)
tigerscore/eval_scripts/mathqa_rate.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_sum.py
"""
import json
import random
import logging
import sys
import numpy as np
import pickle
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from collections import Counter
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json_per_aspect
from xgptscore.constants import EVAL_ASPECTS
logging.basicConfig(level=logging.INFO)

# params
task='data2text'
bart_version="D2T"
dataset="SFHOT"
data_dir="../../BARTScore"
xgptscore_mode="d2t"
version_key=f"{xgptscore_mode}.ref.end_1_5"
our_score_name="xgptscore"
model_name="ChatGPT"
overwrite=False
max_size=200 # set to None to use all examples
num_sys=2
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
input_file=Path(f"{data_dir}/{bart_version}/{dataset}/final_p_with_xgptscore.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")

if not output_file.exists() or overwrite:
    # Load and shuffle data
    logging.info("Loading from {}".format(input_file))
    with open(input_file, "r") as f:
        items = json.load(f)
    if isinstance(max_size, int) and max_size > 0:
        items = items[:max_size]
    # random will cause wrong results
    
    # Data processing
    xgptitems = []
    for item in items:
        item['candidates'] = [
            {
                "model": "reference",
                "decoding_method": "greedy",
                "text": item['output'] if isinstance(item['output'], str) else item['output'][0],
                "scores": {},
            }
        ]
        xgptitems.append(XPGTItem(
            task=task,
            instruction=item['instruction'],
            input=item['input'],
            # ref_output=item['output'],
            ref_output="N/A",
            hypo_output=item['output'] if isinstance(item['output'], str) else item['output'][0],
        ))
    # Run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name,num_workers=5)
    idx = 0
    aspects = EVAL_ASPECTS[task].keys()
    score_dict = {"xgptscore_"+aspect: 0 for aspect in aspects}
    for item in items:
        for cand in item['candidates']:      
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            xgptscore_ans = get_xgptscore_from_json_per_aspect(cand['responses'][-1])
            if xgptscore_ans is None:
                logging.info(f"XGPTScore failed for {cand['text']}")
                # cand['scores']['xgptscore'] = None
            else:
                cand['scores'].update(score_dict)
                cand['scores'].update(xgptscore_ans) 
            idx += 1
        
    # Save results
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading existing results from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)


# by system
# Compute bias
xgptscores = []
for item in items:
    for cand in item['candidates']:
        if our_score_name in cand['scores']:
            xgptscores.append(cand['scores'][our_score_name])

print(f"Mean: {np.mean(xgptscores)}")
print(f"Distribution: {Counter(xgptscores)}")
print(f"Std: {np.std(xgptscores)}")
print(f"Max: {np.min(xgptscores)}")
tigerscore/eval_scripts/test_ref_diff.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_d2t.py
"""
import json
import random
import logging
import sys
import numpy as np
import pickle
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from collections import Counter
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json_per_aspect
from xgptscore.constants import EVAL_ASPECTS
from bs_utils import reformat_d2t_for_bartscore
logging.basicConfig(level=logging.INFO)

# params
task='data2text'
bart_version="D2T"
dataset="BAGEL"
data_dir="../../BARTScore"
xgptscore_mode="d2t"
version_key=f"{xgptscore_mode}.final_instruction_1"
human_score_name="informativeness"
our_score_name="xgptscore"
model_name="ChatGPT"
overwrite=False
max_size=200 # set to None to use all examples
num_sys=2
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
input_file=Path(f"{data_dir}/{bart_version}/{dataset}/final_p_with_xgptscore.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")

if not output_file.exists() or overwrite:
    # Load and shuffle data
    logging.info("Loading from {}".format(input_file))
    with open(input_file, "r") as f:
        items = json.load(f)
    if isinstance(max_size, int) and max_size > 0:
        items = items[:max_size]
    # random will cause wrong results
    
    # Data processing
    xgptitems = []
    for item in items:
        for cand in item['candidates']:
            xgptitems.append(XPGTItem(
                task=task,
                instruction=item['instruction'],
                input=item['input'],
                ref_output=item['output'],
                hypo_output=cand['text']
            ))
    # Run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name,num_workers=5)
    idx = 0
    aspects = EVAL_ASPECTS[task].keys()
    score_dict = {"xgptscore_"+aspect: 0 for aspect in aspects}
    for item in items:
        for cand in item['candidates']:      
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            xgptscore_ans = get_xgptscore_from_json_per_aspect(cand['responses'][-1])
            if xgptscore_ans is None:
                logging.info(f"XGPTScore failed for {cand['text']}")
                # cand['scores']['xgptscore'] = None
            else:
                cand['scores'].update(score_dict)
                cand['scores'].update(xgptscore_ans) 
            idx += 1
        
    # Save results
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading existing results from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)


# by system
# Compute correlation
num_cands = len(items[0]['candidates'])
human_scores = [[cand['scores'][human_score_name] for cand in item['candidates']] for item in items]
human_scores = list(chain(*zip(*human_scores))) # transpose and flatten
metrics = [our_score_name,
    'rouge1_p',
    'rouge2_p',
    'rougel_p',
    'bert_score_p',
    'mover_score',
    'bart_score_cnn_hypo_ref',
    # "bart_score_src_hypo",
    # 'bart_score_para_src_hypo',
    'xgptscore_Fluency',
    'xgptscore_Relevance',
    # 'xgptscore_Informativeness',
    'xgptscore_Accuracy',
]

Pearson_corr = {}
Spearman_corr = {}
Kendall_corr = {}
for metric in metrics:
    metric_scores = [[cand['scores'][metric] if metric in cand['scores'] else None for cand in item['candidates']] for item in items]
    metric_scores = list(chain(*zip(*metric_scores))) # transpose and flatten
    metric_corr = MyCorrelation(num_cands, human_scores, metric_scores)
    Pearson_corr[metric] = metric_corr.Pearson()
    Spearman_corr[metric] = metric_corr.Spearman()
    Kendall_corr[metric] = metric_corr.Kendall()

# sort Corr
Pearson_corr = {k: v for k, v in sorted(Pearson_corr.items(), key=lambda item: item[1][0], reverse=True)}
Spearman_corr = {k: v for k, v in sorted(Spearman_corr.items(), key=lambda item: item[1][0], reverse=True)}
Kendall_corr = {k: v for k, v in sorted(Kendall_corr.items(), key=lambda item: item[1][0], reverse=True)}
Corr_record = {
    "Pearson": Pearson_corr,
    "Spearman": Spearman_corr,
    "Kendall": Kendall_corr,
}
# Save correlation results
corr_results_file = Path("./eval_results/") / f"{bart_version}/{dataset}" / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))
# save to another location
corr_results_file = output_file.parent / "eval_results" / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))

# by segment
d2t_eval_path = Path(f"{output_file}")
reformat_d2t_for_bartscore(d2t_eval_path,our_score_name)
from bs_analysis import D2TStat
d2t_stat = D2TStat(str(d2t_eval_path.with_name(d2t_eval_path.stem + "_bs_format.pkl")))
# Save correlation results
corr_results_file = Path("./eval_results/") / f"{bart_version}/{dataset}" / (output_file.stem + ".bs_corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
d2t_stat.evaluate_text(human_score_name,metrics,corr_results_file)
# summ_stat.get_fact_acc(metrics)
# save to another location
corr_results_file = output_file.parent / "eval_results" / (output_file.stem + ".bs_corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
d2t_stat.evaluate_text(human_score_name,metrics,table=corr_results_file)

# distribution of scores
our_score_list = [[cand['scores'][our_score_name] for cand in item['candidates']] for item in items]
print(f"Our score distribution: {Counter(sorted(list(chain(*our_score_list))))}")
tigerscore/eval_scripts/test_xgptscore_d2t.py
TIGER-AI-Lab/TIGERScore
# test set generate by Ref>ChatGPT>T5-XXL > T5-XL >T5-Large > T5-Base > T5-Small
import json,os,sys
# import pytrec_eval
import logging
from pathlib import Path
from itertools import chain
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from utils import MyCorrelation
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json
logging.basicConfig(level=logging.INFO)
# from common.evaluation import overall_eval

dataset_root_path = "/home//WorkSpace/ExplainableGPTScore_bak/data"
dataset_name = "databricks/databricks-dolly-15k"
dataset_split = "test"
task='instruction-following'
model_list = ["ChatGPT","vicuna-33b-v1.3","vicuna-13b-v1.3","vicuna-7b-v1.3"] # ordered by model size
# not good for long-form QA, T5 series give short answer

dataset_root_path = Path(dataset_root_path)
dataset_file_path = dataset_root_path/dataset_name/f"{dataset_split}_data.json"
dataset_candidates_path = dataset_root_path/dataset_name/"candidates"/dataset_split/"top_p_sampling"
xgptscore_mode="instruction_following"
version_key=f"{xgptscore_mode}.old_2"
middle_file = dataset_root_path/dataset_name/f"rank_eval_mid.json"
output_file = dataset_root_path/dataset_name/f"xgptscore_eval.{version_key}.json"
overwrite = False
max_size = 200
model_name="ChatGPT"

if not middle_file.exists():
    # load dataset
    datas = json.load(open(dataset_file_path))
    # for item in datas:
    #     item["candidates"] = [
    #         {
    #             "text":item["output"] if isinstance(item["output"],str) else item["output"][0],
    #             "source":"ref"
    #         }
    #     ]
    for item in datas:
        item["candidates"] = []
    data_dict = {item["id"]:item for item in datas}
    # load candidates

    #TODO: we choose the cand with highest BARTScore
    for i,_model_name in enumerate(model_list):
        model_data = json.load(open(dataset_candidates_path /f"{_model_name}.eval.json"))
        for item in model_data:
            cands = sorted(item["candidates"], key=lambda x:x["scores"]["bart_score"], reverse=True)
            cands[0]["scores"]["rank"] = -i
            data_dict[item["id"]]["candidates"].append({
                "text":cands[0]["text"],
                "source":_model_name,
                "scores":cands[0]["scores"]
            })

    items = [data_dict[item_id] for item_id in data_dict.keys()]
    candidates = [[candidate["text"] for candidate in data["candidates"] ]for data in items]
    
    # targets = [data["output"] for data in items]
    # metrics = ["bleu", "rouge1","rouge2","rougeL","bart_score","bart_score_cnn"]
    # scores = overall_eval(candidates, targets, metrics, 1)
    # for metric in metrics:
    #     scores_metric = scores[metric]
    #     for i, sample in enumerate(items):
    #         _sample_cands = sample['candidates']
    #         for j, _sample_cand in enumerate(_sample_cands):
    #             _sample_cand['scores'][metric] = scores_metric[i][j]
    with open(middle_file, "w") as f:
        json.dump(items, f, indent=4)
        logging.info("Saved to {}".format(middle_file))
else:
    with open(middle_file, "r") as f:
        items = json.load(f)
    
# evaluate XGPTScore
if not output_file.exists() or overwrite:
    # eval_by_other_metrics
    # Data processing
    xgptitems = []
    for item in items:
        for cand in item['candidates']:
            xgptitems.append(XPGTItem(
                task=task,
                instruction=item['instruction'],
                input=item['input'],
                ref_output=item['output'],
                hypo_output=cand['text']
            ))
    # Run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name)
    idx = 0
    for item in items:
        for cand in item['candidates']:
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            cand['scores']['xgptscore'] = get_xgptscore_from_json(cand['responses'][-1])
            idx += 1
        
    # Save results
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading existing results from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)
        
# evaluate system



num_cands = len(items[0]['candidates'])
human_scores = [[cand['scores']["rank"] for cand in item['candidates']] for item in items]
human_scores = list(chain(*zip(*human_scores))) # transpose and flatten
metrics = ["xgptscore","bleu","rouge1","rouge2","rougeL","rougeLsum","bart_score","bart_score_cnn"]
# metrics = ["xgptscore"]

Pearson_corr = {}
Spearman_corr = {}
Kendall_corr = {}
for metric in metrics:
    metric_scores = [[cand['scores'][metric] for cand in item['candidates']] for item in items]
    metric_scores = list(chain(*zip(*metric_scores))) # transpose and flatten
    metric_corr = MyCorrelation(num_cands, human_scores, metric_scores)
    Pearson_corr[metric] = metric_corr.Pearson()
    Spearman_corr[metric] = metric_corr.Spearman()
    Kendall_corr[metric] = metric_corr.Kendall()

# sort Corr
Pearson_corr = {k: v for k, v in sorted(Pearson_corr.items(), key=lambda item: item[1][0], reverse=True)}
Spearman_corr = {k: v for k, v in sorted(Spearman_corr.items(), key=lambda item: item[1][0], reverse=True)}
Kendall_corr = {k: v for k, v in sorted(Kendall_corr.items(), key=lambda item: item[1][0], reverse=True)}
Corr_record = {
    "Pearson": Pearson_corr,
    "Spearman": Spearman_corr,
    "Kendall": Kendall_corr,
}
# Save correlation results
corr_results_file = Path("./eval_results/") / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))
# save to another location
corr_results_file = output_file.parent / "eval_results" / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))
# print("Correlation results:")
# print(json.dumps(Corr_record, indent=4, ensure_ascii=False))
tigerscore/eval_scripts/test_xgptscore_instruction.py
TIGER-AI-Lab/TIGERScore
# test set generate by Ref>ChatGPT>T5-XXL > T5-XL >T5-Large > T5-Base > T5-Small
import json,os,sys
# import pytrec_eval
import logging
from pathlib import Path
from itertools import chain
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from utils import MyCorrelation
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json
logging.basicConfig(level=logging.INFO)
# from common.evaluation import overall_eval

dataset_root_path = "/home//WorkSpace/ExplainableGPTScore_bak/data"
dataset_name = "eli5"#"cosmos_qa""eli5""din0s/asqa""DongfuTingle/FeTaQA"
dataset_split = "test"
task='long-form QA'
model_list = ["ChatGPT","vicuna-33b-v1.3","vicuna-13b-v1.3","vicuna-7b-v1.3"] # ordered by model size
# not good for long-form QA, T5 series give short answer

dataset_root_path = Path(dataset_root_path)
dataset_file_path = dataset_root_path/dataset_name/f"{dataset_split}_data.json"
dataset_candidates_path = dataset_root_path/dataset_name/"candidates"/dataset_split/"top_p_sampling"
xgptscore_mode="longform_qa"
version_key=f"{xgptscore_mode}.old_2_new_as"
# middle_file = dataset_root_path/dataset_name/f"rank_eval_mid.json"
middle_file = Path("/home//WorkSpace/ExplainableGPTScore_bak/data/lfqa/test.json")
output_file = Path(f"/home//WorkSpace/ExplainableGPTScore_bak/data/lfqa/xgptscore_eval.{version_key}.json")
overwrite = False
max_size = 200
model_name="ChatGPT"

if not middle_file.exists():
    # load dataset
    datas = json.load(open(dataset_file_path))
    # for item in datas:
    #     item["candidates"] = [
    #         {
    #             "text":item["output"] if isinstance(item["output"],str) else item["output"][0],
    #             "source":"ref"
    #         }
    #     ]
    for item in datas:
        item["candidates"] = []
    data_dict = {item["id"]:item for item in datas}
    # load candidates

    for i,_model_name in enumerate(model_list):
        model_data = json.load(open(dataset_candidates_path /f"{_model_name}.eval.json"))
        for item in model_data:
            cands = sorted(item["candidates"], key=lambda x:x["scores"]["bart_score"], reverse=True)
            cands[0]["scores"]["rank"] = -i
            data_dict[item["id"]]["candidates"].append({
                "text":cands[0]["text"],
                "source":_model_name,
                "scores":cands[0]["scores"]
            })

    items = [data_dict[item_id] for item_id in data_dict.keys()]
    candidates = [[candidate["text"] for candidate in data["candidates"] ]for data in items]
    
    # targets = [data["output"] for data in items]
    # metrics = ["bleu", "rouge1","rouge2","rougeL","bart_score","bart_score_cnn"]
    # scores = overall_eval(candidates, targets, metrics, 1)
    # for metric in metrics:
    #     scores_metric = scores[metric]
    #     for i, sample in enumerate(items):
    #         _sample_cands = sample['candidates']
    #         for j, _sample_cand in enumerate(_sample_cands):
    #             _sample_cand['scores'][metric] = scores_metric[i][j]
    with open(middle_file, "w") as f:
        json.dump(items, f, indent=4)
        logging.info("Saved to {}".format(middle_file))
else:
    with open(middle_file, "r") as f:
        items = json.load(f)
    
# evaluate XGPTScore
if not output_file.exists() or overwrite:
    # eval_by_other_metrics
    # Data processing
    xgptitems = []
    for item in items:
        for cand in item['candidates']:
            xgptitems.append(XPGTItem(
                task=task,
                instruction=item['instruction'],
                input=item['input'],
                ref_output=item['output'],
                hypo_output=cand['text']
            ))
    # Run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name)
    idx = 0
    for item in items:
        for cand in item['candidates']:
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            cand['scores']['xgptscore'] = get_xgptscore_from_json(cand['responses'][-1])
            idx += 1
        
    # Save results
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading existing results from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)
        
# evaluate system



num_cands = len(items[0]['candidates'])
human_scores = [[cand['scores']["rank"] for cand in item['candidates']] for item in items]
human_scores = list(chain(*zip(*human_scores))) # transpose and flatten
metrics = ["xgptscore","bleu","rouge1","rouge2","rougeL","rougeLsum","bart_score","bart_score_cnn"]
# metrics = ["xgptscore"]

Pearson_corr = {}
Spearman_corr = {}
Kendall_corr = {}
for metric in metrics:
    metric_scores = [[cand['scores'][metric] for cand in item['candidates']] for item in items]
    metric_scores = list(chain(*zip(*metric_scores))) # transpose and flatten
    metric_corr = MyCorrelation(num_cands, human_scores, metric_scores)
    Pearson_corr[metric] = metric_corr.Pearson()
    Spearman_corr[metric] = metric_corr.Spearman()
    Kendall_corr[metric] = metric_corr.Kendall()

# sort Corr
Pearson_corr = {k: v for k, v in sorted(Pearson_corr.items(), key=lambda item: item[1][0], reverse=True)}
Spearman_corr = {k: v for k, v in sorted(Spearman_corr.items(), key=lambda item: item[1][0], reverse=True)}
Kendall_corr = {k: v for k, v in sorted(Kendall_corr.items(), key=lambda item: item[1][0], reverse=True)}
Corr_record = {
    "Pearson": Pearson_corr,
    "Spearman": Spearman_corr,
    "Kendall": Kendall_corr,
}
# Save correlation results
corr_results_file = Path("./eval_results/") / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))
# save to another location
corr_results_file = output_file.parent / "eval_results" / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))
# print("Correlation results:")
# print(json.dumps(Corr_record, indent=4, ensure_ascii=False))
tigerscore/eval_scripts/test_xgptscore_longform.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_d2t.py
"""
import json
import random
import logging
import sys
import numpy as np
import pickle
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from collections import Counter
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json
from xgptscore.constants import EVAL_ASPECTS
logging.basicConfig(level=logging.INFO)

# params
task='mathQA'
xgptscore_mode="longform_qa"
version_key=f"{xgptscore_mode}.test1"
human_score_name="accuracy"
our_score_name="xgptscore"
model_name="gpt-4"
overwrite=False
max_size=200 # set to None to use all examples
num_sys=2
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
input_file=Path("/home//WorkSpace/ExplainableGPTScore_bak/data/mathqa/gsm8k_test_output_prepared.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")

if not output_file.exists() or overwrite:
    # Load and shuffle data
    logging.info("Loading from {}".format(input_file))
    with open(input_file, "r") as f:
        items = json.load(f)
    if isinstance(max_size, int) and max_size > 0:
        items = items[:max_size]
    # random will cause wrong results
    
    # Data processing
    xgptitems = []
    for item in items:
        for cand in item['candidates']:
            xgptitems.append(XPGTItem(
                task=task,
                instruction=item['instruction'],
                input=item['input'],
                ref_output=item['output'],
                hypo_output=cand['text']
            ))
    # Run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name,num_workers=5)
    idx = 0
    for item in items:
        for cand in item['candidates']:      
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            cand['scores']['xgptscore'] = get_xgptscore_from_json(cand['responses'][-1])
            idx += 1
        
    # Save results
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading existing results from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)


# by system
# Compute correlation
num_cands = len(items[0]['candidates'])
human_scores = [[cand['scores'][human_score_name] for cand in item['candidates']] for item in items]
human_scores = list(chain(*zip(*human_scores))) # transpose and flatten
metrics = [our_score_name,
    # # 'rouge1_p',
    # # 'rouge2_p',
    # # 'rougel_p',
    # # 'bert_score_p',
    # # 'mover_score',
    # # 'bart_score_cnn_hypo_ref',
    # # "bart_score_src_hypo",
    # # 'bart_score_para_src_hypo',
    # 'xgptscore_Fluency',
    # 'xgptscore_Relevance',
    # # 'xgptscore_Informativeness',
    # 'xgptscore_Accuracy',
]

Pearson_corr = {}
Spearman_corr = {}
Kendall_corr = {}
for metric in metrics:
    metric_scores = [[cand['scores'][metric] if metric in cand['scores'] else None for cand in item['candidates']] for item in items]
    metric_scores = list(chain(*zip(*metric_scores))) # transpose and flatten
    metric_corr = MyCorrelation(num_cands, human_scores, metric_scores)
    Pearson_corr[metric] = metric_corr.Pearson()
    Spearman_corr[metric] = metric_corr.Spearman()
    Kendall_corr[metric] = metric_corr.Kendall()

# sort Corr
Pearson_corr = {k: v for k, v in sorted(Pearson_corr.items(), key=lambda item: item[1][0], reverse=True)}
Spearman_corr = {k: v for k, v in sorted(Spearman_corr.items(), key=lambda item: item[1][0], reverse=True)}
Kendall_corr = {k: v for k, v in sorted(Kendall_corr.items(), key=lambda item: item[1][0], reverse=True)}
Corr_record = {
    "Pearson": Pearson_corr,
    "Spearman": Spearman_corr,
    "Kendall": Kendall_corr,
}
# Save correlation results
corr_results_file = Path("./eval_results/") / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))
# save to another location
corr_results_file = output_file.parent / "eval_results" / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))

tigerscore/eval_scripts/test_xgptscore_mathqa.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_sum.py
"""
import json
import random
import logging
import sys
import numpy as np
import pickle
from pathlib import Path
from utils import MyCorrelation
from collections import Counter
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json_per_aspect
from xgptscore.constants import EVAL_ASPECTS
from bs_utils import reformat_sum_for_bartscore
logging.basicConfig(level=logging.INFO)

# params
task='summarization'
bart_version="SUM"
dataset="SummEval"
data_dir="../../BARTScore"
xgptscore_mode="align_score"
version_key=f"{xgptscore_mode}.new_new_end"
human_score_name="relevance"
our_score_name="xgptscore"
model_name="ChatGPT"
overwrite=False
max_size=200 # set to None to use all examples
num_sys=2
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
input_file=Path(f"{data_dir}/{bart_version}/{dataset}/final_p_with_xgptscore.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")

if not output_file.exists() or overwrite:
    # Load and shuffle data
    logging.info("Loading from {}".format(input_file))
    with open(input_file, "r") as f:
        items = json.load(f)
    if isinstance(max_size, int) and max_size > 0:
        items = items[:max_size]
    # random will cause wrong results
    
    # Data processing
    xgptitems = []
    for item in items:
        for cand in item['candidates']:
            xgptitems.append(XPGTItem(
                task=task,
                instruction=item['instruction'],
                input=item['input'],
                ref_output=item['output'],
                hypo_output=cand['text']
            ))
    # Run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name,num_workers=5)
    idx = 0
    aspects = EVAL_ASPECTS[task].keys()
    score_dict = {"xgptscore_"+aspect: 0 for aspect in aspects}
    for item in items:
        for cand in item['candidates']:      
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            xgptscore_ans = get_xgptscore_from_json_per_aspect(cand['responses'][-1])
            if xgptscore_ans is None:
                logging.info(f"XGPTScore failed for {cand['text']}")
                # cand['scores']['xgptscore'] = None
            else:
                cand['scores'].update(score_dict)
                cand['scores'].update(xgptscore_ans) 
            idx += 1
        
    # Save results
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading existing results from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)


# by system
# Compute correlation
num_cands = len(items[0]['candidates'])
human_scores = [[cand['scores'][human_score_name] for cand in item['candidates']] for item in items]
human_scores = list(chain(*zip(*human_scores))) # transpose and flatten
metrics = [our_score_name,
    'rouge1_r',
    'rouge2_r',
    'rougel_r',
    'bert_score_r',
    'mover_score',
    'prism_hypo_ref',
    "prism_src_hypo",
    'bart_score_cnn_hypo_ref',
    "bart_score_src_hypo",
    'bart_score_para_src_hypo',
    'xgptscore',
    'xgptscore_Fluency',
    'xgptscore_Relevance',
    'xgptscore_Coherence',
    'xgptscore_Consistency',
]

Pearson_corr = {}
Spearman_corr = {}
Kendall_corr = {}
for metric in metrics:
    metric_scores = [[cand['scores'][metric] if metric in cand['scores'] else None for cand in item['candidates']] for item in items]
    metric_scores = list(chain(*zip(*metric_scores))) # transpose and flatten
    metric_corr = MyCorrelation(num_cands, human_scores, metric_scores)
    Pearson_corr[metric] = metric_corr.Pearson()
    Spearman_corr[metric] = metric_corr.Spearman()
    Kendall_corr[metric] = metric_corr.Kendall()

# sort Corr
Pearson_corr = {k: v for k, v in sorted(Pearson_corr.items(), key=lambda item: item[1][0], reverse=True)}
Spearman_corr = {k: v for k, v in sorted(Spearman_corr.items(), key=lambda item: item[1][0], reverse=True)}
Kendall_corr = {k: v for k, v in sorted(Kendall_corr.items(), key=lambda item: item[1][0], reverse=True)}
Corr_record = {
    "Pearson": Pearson_corr,
    "Spearman": Spearman_corr,
    "Kendall": Kendall_corr,
}
# Save correlation results
corr_results_file = Path("./eval_results/") / f"{bart_version}/{dataset}" / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))
# save to another location
corr_results_file = output_file.parent / "eval_results" / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))

# by segment
sum_eval_path = Path(f"{output_file}")
reformat_sum_for_bartscore(sum_eval_path,our_score_name)
from bs_analysis import SUMStat
summ_stat = SUMStat(str(sum_eval_path.with_name(sum_eval_path.stem + "_bs_format.pkl")))
# Save correlation results
corr_results_file = Path("./eval_results/") / f"{bart_version}/{dataset}" / (output_file.stem + ".bs_corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
summ_stat.evaluate_summary(human_score_name,metrics,table=corr_results_file)
# summ_stat.get_fact_acc(metrics)
# save to another location
corr_results_file = output_file.parent / "eval_results" / (output_file.stem + ".bs_corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
summ_stat.evaluate_summary(human_score_name,metrics,table=corr_results_file)
# summ_stat.get_fact_acc()

# distribution of scores
our_score_list = [[cand['scores'][our_score_name] for cand in item['candidates']] for item in items]
print(f"Our score distribution: {Counter(sorted(list(chain(*our_score_list))))}")
tigerscore/eval_scripts/test_xgptscore_sum.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_d2t.py
"""
import json
import random
import logging
import sys
import numpy as np
import pickle
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json_per_aspect
from xgptscore.constants import EVAL_ASPECTS
from bs_utils import reformat_d2t_for_bartscore
logging.basicConfig(level=logging.INFO)

# params
task='data2text'
data_dir="../../data/webnlg"
xgptscore_mode="d2t_wmt_mqm"
version_key=f"{xgptscore_mode}.simple"
human_score_name="Relevance"
our_score_name="xgptscore"
model_name="ChatGPT"
overwrite=False
max_size=200 # set to None to use all examples
num_sys=2
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
input_file=Path(f"{data_dir}/webnlg2020_gen_with_scores.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")

if not output_file.exists() or overwrite:
    # Load and shuffle data
    logging.info("Loading from {}".format(input_file))
    with open(input_file, "r") as f:
        items = json.load(f)
    if isinstance(max_size, int) and max_size > 0:
        items = items[:max_size]
    # random will cause wrong results
    
    # Data processing
    xgptitems = []
    for item in items:
        for cand in item['candidates']:
            xgptitems.append(XPGTItem(
                task=task,
                instruction=item['instruction'],
                input=item['input'],
                ref_output=item['output'],
                hypo_output=cand['text']
            ))
    # Run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name,num_workers=5)
    idx = 0
    aspects = EVAL_ASPECTS[task].keys()
    score_dict = {"xgptscore_"+aspect: 0 for aspect in aspects}
    for item in items:
        for cand in item['candidates']:      
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            xgptscore_ans = get_xgptscore_from_json_per_aspect(cand['responses'][-1])
            if xgptscore_ans is None:
                logging.info(f"XGPTScore failed for {cand['text']}")
                cand['scores']['xgptscore'] = None
            else:
                cand['scores'].update(score_dict)
                cand['scores'].update(xgptscore_ans) 
            idx += 1
        
    # Save results
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading existing results from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)


# by system
# Compute correlation
num_cands = len(items[0]['candidates'])
# human_scores = [[cand['scores'][human_score_name] for cand in item['candidates']] for item in items]
# human_scores = list(chain(*zip(*human_scores))) # transpose and flatten
metrics = [our_score_name,
    # 'rouge1_p',
    # 'rouge2_p',
    # 'rougel_p',
    # 'bert_score_p',
    # 'mover_score',
    # 'bart_score_cnn_hypo_ref',
    # "bart_score_src_hypo",
    # 'bart_score_para_src_hypo',
    "bleu",
    "rouge1",
    "rouge2",
    "rougeL",
    "bart_score",
    "bart_score_cnn",
    'xgptscore_Fluency',
    'xgptscore_Relevance',
    'xgptscore_Accuracy',
    # 'xgptscore_Consistency',
]

# Pearson_corr = {}
# Spearman_corr = {}
# Kendall_corr = {}
# for metric in metrics:
#     metric_scores = [[cand['scores'][metric] if metric in cand['scores'] else None for cand in item['candidates']] for item in items]
#     metric_scores = list(chain(*zip(*metric_scores))) # transpose and flatten
#     metric_corr = MyCorrelation(num_cands, human_scores, metric_scores)
#     Pearson_corr[metric] = metric_corr.Pearson()
#     Spearman_corr[metric] = metric_corr.Spearman()
#     Kendall_corr[metric] = metric_corr.Kendall()

# # sort Corr
# Pearson_corr = {k: v for k, v in sorted(Pearson_corr.items(), key=lambda item: item[1][0], reverse=True)}
# Spearman_corr = {k: v for k, v in sorted(Spearman_corr.items(), key=lambda item: item[1][0], reverse=True)}
# Kendall_corr = {k: v for k, v in sorted(Kendall_corr.items(), key=lambda item: item[1][0], reverse=True)}
# Corr_record = {
#     "Pearson": Pearson_corr,
#     "Spearman": Spearman_corr,
#     "Kendall": Kendall_corr,
# }
# # Save correlation results
# corr_results_file = Path("./eval_results/")  / (output_file.stem + ".corr.json")
# corr_results_file.parent.mkdir(parents=True, exist_ok=True)
# with open(corr_results_file, "w") as f:
#     json.dump(Corr_record, f, indent=4, ensure_ascii=False)
#     logging.info("Saved to {}".format(corr_results_file))
# # save to another location
# corr_results_file = output_file.parent / "eval_results" / (output_file.stem + ".corr.json")
# corr_results_file.parent.mkdir(parents=True, exist_ok=True)
# with open(corr_results_file, "w") as f:
#     json.dump(Corr_record, f, indent=4, ensure_ascii=False)
#     logging.info("Saved to {}".format(corr_results_file))

# by segment
d2t_eval_path = Path(f"{output_file}")
reformat_d2t_for_bartscore(d2t_eval_path,our_score_name)
from bs_analysis import D2TStat
d2t_stat = D2TStat(str(d2t_eval_path.with_name(d2t_eval_path.stem + "_bs_format.pkl")))
# Save correlation results
corr_results_file = Path("./eval_results/")  / (output_file.stem + ".bs_corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
d2t_stat.evaluate_text(human_score_name,metrics,corr_results_file)
# summ_stat.get_fact_acc(metrics)
# save to another location
corr_results_file = output_file.parent / "eval_results" / (output_file.stem + ".bs_corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
d2t_stat.evaluate_text(human_score_name,metrics,table=corr_results_file)

tigerscore/eval_scripts/test_xgptscore_webnlg.py
TIGER-AI-Lab/TIGERScore
"""
Usage:
    python test_xgptscore_wmt.py
"""
import json
import logging
import sys
import numpy as np
from pathlib import Path
from utils import MyCorrelation
sys.path.append(str(Path(__file__).parent.parent))
from xgptscore.xgptscore import xgptscore
from itertools import chain
from xgptscore.process_utils import XPGTItem, get_xgptscore_from_json
logging.basicConfig(level=logging.INFO)

# params
task='translation'
wmt_version="wmt22"
lang_pair="en-ru"
data_dir=f"../../data/"
xgptscore_mode="old_ea"
version_key=f"{xgptscore_mode}.random_2_sys_new_wmt_mqm"
human_score_name="mqm"
model_name="ChatGPT"
overwrite=True
max_size=25 # set to None to use all examples
num_sys=2
if isinstance(max_size, int) and max_size > 0:
    version_key = f"{version_key}_{max_size}"

# load data
input_file=Path(f"{data_dir}/{wmt_version}/{lang_pair}/eval_data.json")
if version_key:
    output_file = input_file.with_suffix(f".{version_key}.json")
else:
    output_file = input_file.with_suffix(f".default.json")

if not output_file.exists() or overwrite:
    # Load and shuffle data
    logging.info("Loading from {}".format(input_file))
    with open(input_file, "r") as f:
        items = json.load(f)
    np.random.seed(42)
    np.random.shuffle(items)
    if isinstance(max_size, int) and max_size > 0:
        items = items[:max_size]
    
    # randomly select 2 systems
    if isinstance(num_sys, int) and num_sys > 0:
        """Randomly select num_sys systems for each example"""
        # all_sys_names = set([x['model'] for x in items[0]['candidates']])
        # shuffled_sys_names = list(all_sys_names)
        # np.random.seed(42)
        # np.random.shuffle(shuffled_sys_names)
        # sub_sampled_sys = shuffled_sys_names[:num_sys]
        # for item in items:
        #     sys_cand_map = {x['model']: x for x in item['candidates']}
        #     item['candidates'] = [sys_cand_map[x] for x in sub_sampled_sys]
        """Randomly select num_sys candidates for each example"""
        for item in items:
            idxs = np.random.permutation(len(item['candidates']))[:num_sys]
            item['candidates'] = [item['candidates'][idx] for idx in idxs]
    
    # Data processing
    xgptitems = []
    for item in items:
        for cand in item['candidates']:
            xgptitems.append(XPGTItem(
                task=task,
                instruction=item['instruction'],
                input=item['input'],
                ref_output=item['refs'],
                hypo_output=cand['text']
            ))

    # Run xgptscore
    result = xgptscore(xgptitems, mode=xgptscore_mode, model_name=model_name)
    idx = 0
    for item in items:
        for cand in item['candidates']:
            cand['responses'] = result['round_completions'][idx]
            cand['messages_records'] = result['messages_records'][idx]
            cand['scores']['xgptscore'] = get_xgptscore_from_json(cand['responses'][-1])
            idx += 1
        
    # Save results
    with open(output_file, "w") as f:
        json.dump(items, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(output_file))
else:
    logging.info("Loading existing results from {}".format(output_file))
    with open(output_file, "r") as f:
        items = json.load(f)

# Compute correlation
num_cands = len(items[0]['candidates'])
human_scores = [[cand['scores'][human_score_name] for cand in item['candidates']] for item in items]
human_scores = list(chain(*zip(*human_scores))) # transpose and flatten
metrics = ["xgptscore", "BLEU-refA", "BERTScore-refA", "COMET-22-refA", "COMET-20-refA", "BLEURT-20-refA", "chrF-refA"]

Pearson_corr = {}
Spearman_corr = {}
Kendall_corr = {}
for metric in metrics:
    metric_scores = [[cand['scores'][metric] for cand in item['candidates']] for item in items]
    metric_scores = list(chain(*zip(*metric_scores))) # transpose and flatten
    metric_corr = MyCorrelation(num_cands, human_scores, metric_scores)
    Pearson_corr[metric] = metric_corr.Pearson()
    Spearman_corr[metric] = metric_corr.Spearman()
    Kendall_corr[metric] = metric_corr.Kendall()

# sort Corr
Pearson_corr = {k: v for k, v in sorted(Pearson_corr.items(), key=lambda item: item[1][0], reverse=True)}
Spearman_corr = {k: v for k, v in sorted(Spearman_corr.items(), key=lambda item: item[1][0], reverse=True)}
Kendall_corr = {k: v for k, v in sorted(Kendall_corr.items(), key=lambda item: item[1][0], reverse=True)}
Corr_record = {
    "Pearson": Pearson_corr,
    "Spearman": Spearman_corr,
    "Kendall": Kendall_corr,
}
# Save correlation results
corr_results_file = Path("./eval_results/") / f"{wmt_version}/{lang_pair}" / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))
# save to another location
corr_results_file = output_file.parent / "eval_results" / (output_file.stem + ".corr.json")
corr_results_file.parent.mkdir(parents=True, exist_ok=True)
with open(corr_results_file, "w") as f:
    json.dump(Corr_record, f, indent=4, ensure_ascii=False)
    logging.info("Saved to {}".format(corr_results_file))
tigerscore/eval_scripts/test_xgptscore_wmt.py
TIGER-AI-Lab/TIGERScore
import logging
from mt_metrics_eval.stats import Correlation
from typing import List
class MyCorrelation(Correlation):
    def __init__(self, num_sys:int, gold_scores:List[int], metric_scores:List[int]):
        # remove nan in metrics scores
        none_metric_scores_idxs = [idx for idx, x in enumerate(metric_scores) if x is None]
        logging.info("Remove {} nan scores from {} scores".format(
            len(none_metric_scores_idxs),
            len(metric_scores)
        ))
        gold_scores = gold_scores.copy()
        # set gold scores to None if metric scores are None
        for idx in none_metric_scores_idxs[::-1]:
            gold_scores[idx] = None 
        super().__init__(num_sys, gold_scores, metric_scores)
        

tigerscore/eval_scripts/utils.py
TIGER-AI-Lab/TIGERScore
# Example usage
import os
import sys
import fire
import json
from pathlib import Path
os.environ["OPENAI_API_KEY"] = ""
os.environ["OPENAI_API_BASE"] = ""
os.environ["OPENAI_API_TYPE"] = "azure"
os.environ["OPENAI_API_VERSION"] = "2023-03-15-preview"
sys.path.append("../")
from xgptscore.openai_utils import openai_completions, _chatml_to_prompt
from xgptscore.constants import EVAL_ASPECTS
from string import Template

TEMPLATE = """\
You are evaluating an ${task} task. Some errors in an incorrect output could be attributed to the following aspects:
${aspects_descriptions}

Please elaborate 10 specific error types for each aspect above. Each error type should represent a specific error that falls under the aspect. Error types should be mutually exclusive and collectively exhaustive.\
"""


def main(
    task: str,
):
    
    task_aspects = EVAL_ASPECTS[task]
    prompt = Template(TEMPLATE).substitute(
        task=task,
        aspects_descriptions="\n".join([f"- {aspect}: {description}" for aspect, description in task_aspects.items()])
    )
    prompts = [prompt]
    chatmls = [[{"role":"system","content":"You are an AI assistant that helps people find information."},
                {"role":"user","content": prompt}] for prompt in prompts[:1]]

    chatml_prompts = [_chatml_to_prompt(chatml) for chatml in chatmls]
    results = openai_completions(chatml_prompts, model_name="gpt-4")
    output_file = Path("./error_types/" + task + ".txt")
    output_file.parent.mkdir(parents=True, exist_ok=True)
    results['propmts'] = prompts
    with open(output_file, "w") as f:
        json.dump(results, f, indent=4, ensure_ascii=False)

if __name__ == "__main__":
    fire.Fire(main)
tigerscore/get_error_types/get_error_types.py
TIGER-AI-Lab/TIGERScore
import json
import argparse
import logging
import numpy as np
from pathlib import Path
from collections import Counter
from itertools import chain
from transformers import AutoTokenizer
logging.basicConfig(level=logging.INFO)
"""
Datasets:
    - Webnlg 20000
    - wikitables 10000
    - totto 10000
"""
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='../../data')
    args = parser.parse_args()
    args.data_dir = Path(args.data_dir)
    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
    agg_data = []
    datasets = ["totto","kasnerz/wikitabletext","webnlg"]
    logging.info("Aggregating data from {}".format(args.data_dir))
    np.random.seed(42)
    for dataset in datasets:
        with open(args.data_dir /"{}".format(dataset) /"train_data_prepared.json", "r") as f:
            data = json.load(f)
            data = [x for x in data if len(x['candidates']) > 0]
            data = [x for x in data if len(tokenizer.encode(x['input'], add_special_tokens=False)) < 600]
            np.random.shuffle(data)
            # give candidates
            if dataset in ["totto","kasnerz/wikitabletext"]:
                # data = list(np.random.choice(data, 3000, replace=False))
                # per 5
                # data = list(np.random.choice(data, len(data)//3*2,replace=False))
                for i, item in enumerate(data):
                    new_cands = sorted(item['candidates'], key=lambda x: x['scores']['bart_score'], reverse=True)
                    if i < len(data)//3 :
                        item['candidates'] = [new_cands[0]]
                    elif i < len(data)//3 * 2:
                        item['candidates'] = [new_cands[2]]
                    else:
                        item['candidates'] = [new_cands[-1]]
            if dataset == "webnlg":
                for i, item in enumerate(data):
                    # for can in item['candidates']:
                    #     can['scores'].update({"human_score": can['scores']["Correctness"] + can['scores']["Fluency"] + can['scores']["Relevance"] + can['scores']["TextStructure"] + can['scores']["DataCoverage"]})
                    # new_cands = sorted(item['candidates'], key=lambda x: x['scores']["human_score"], reverse=True)
                    new_cands = sorted(item['candidates'], key=lambda x: x['scores']["bart_score"], reverse=True)
                    # if i < len(data)//3 :
                    #     item['candidates'] = [new_cands[0],new_cands[len(new_cands)//2]]
                    # elif i < len(data)//3 * 2:
                    #     item['candidates'] = [new_cands[0],new_cands[-1]]
                    # else:
                    #     item['candidates'] = [new_cands[len(new_cands)//2],new_cands[-1]]
     
                    item['candidates'] = [new_cands[0],new_cands[len(new_cands)//2],new_cands[-1]]
                np.random.shuffle(data)
                # data = list(np.random.choice(data, 2000, replace=False))
            # data = list(np.random.choice(data, len(data)//2,replace=False))
            logging.info("Loaded {} data from {}".format(len(data), args.data_dir /"{}".format(dataset) /"train_data_prepared.json"))
            logging.info("Outputs: {}".format(np.sum([len(x['candidates']) for x in data])))
            for x in data:
                x['dataset'] = dataset
            agg_data += data
    logging.info("# Total aggregated # {} data for data to text".format(len(agg_data)))
    logging.info("# Outputs: {}".format(np.sum([len(x['candidates']) for x in agg_data])))
    # length statistics
    input_lens = [len(tokenizer.encode(x['input'], add_special_tokens=False)) for x in agg_data]
    cand_lens = [[len(tokenizer.encode(cand['text'], add_special_tokens=False)) for cand in x['candidates']] for x in agg_data]
    cand_lens = list(chain(*cand_lens))
    if dataset == "webnlg":
        ref_lens = [[len(tokenizer.encode(ref, add_special_tokens=False)) for ref in x['output']] for x in agg_data]
        ref_lens = list(chain(*ref_lens))
    else:
        ref_lens = [len(tokenizer.encode(x['output'], add_special_tokens=False))  for x in agg_data]
    logging.info("Input length statistics:")
    logging.info("  Avg/Max/90%/50%: {}/{}/{}/{}".format(np.mean(input_lens), np.max(input_lens), np.percentile(input_lens, 90), np.percentile(input_lens, 50)))
    logging.info("Candidate length statistics:")
    logging.info("  Avg/Max/90%: {}/{}/{}".format(np.mean(cand_lens), np.max(cand_lens), np.percentile(cand_lens, 90)))
    logging.info("Reference length statistics:")
    logging.info("  Avg/Max/90%: {}/{}/{}".format(np.mean(ref_lens), np.max(ref_lens), np.percentile(ref_lens, 90)))
    
    output_file = args.data_dir /"d2t" / "train_data_new.json"
    with open(output_file, "w") as f:
        json.dump(agg_data, f, indent=4, ensure_ascii=False)
        logging.info("Saved aggregated data to {}".format(output_file))







tigerscore/prepare_data/aggregate_d2t_train_data.py
TIGER-AI-Lab/TIGERScore
import json
import argparse
import logging
import numpy as np
from pathlib import Path
from collections import Counter
from itertools import chain
from transformers import AutoTokenizer
logging.basicConfig(level=logging.INFO)
"""
Datasets:
    - mix-instruction
"""
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='../../data')
    args = parser.parse_args()
    args.data_dir = Path(args.data_dir)
    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
    agg_data = []
    splits = ["test","validation"]
    logging.info("Aggregating data from {}".format(args.data_dir))
    np.random.seed(42)
    for split in splits:
        with open(args.data_dir /"llm-blender/mix-instruct" /f"{split}_data_prepared.json", "r") as f:
            data = json.load(f)
            data = [x for x in data if len(x['candidates']) > 0]
            data = [x for x in data if len(tokenizer.encode(x['input'], add_special_tokens=False)) < 600]
            np.random.shuffle(data)
            # give candidates
                # data = list(np.random.choice(data, 3000, replace=False))
                # per 5
                # data = list(np.random.choice(data, len(data)//3*2,replace=False))
            for i, item in enumerate(data):
                new_cands = sorted(item['candidates'], key=lambda x: x['scores']['bartscore'], reverse=True)
                if i < len(data)//3 :
                    item['candidates'] = [new_cands[0]]
                elif i < len(data)//3 * 2:
                    item['candidates'] = [new_cands[len(new_cands)//2]]
                else:
                    item['candidates'] = [new_cands[-1]]
            np.random.shuffle(data)
                # data = list(np.random.choice(data, 2000, replace=False))
            # data = list(np.random.choice(data, len(data)//2,replace=False))
            logging.info("Outputs: {}".format(np.sum([len(x['candidates']) for x in data])))
            agg_data += data
    logging.info("# Total aggregated # {} data for data to text".format(len(agg_data)))
    logging.info("# Outputs: {}".format(np.sum([len(x['candidates']) for x in agg_data])))
    # length statistics
    input_lens = [len(tokenizer.encode(x['input'], add_special_tokens=False)) for x in agg_data]
    cand_lens = [[len(tokenizer.encode(cand['text'], add_special_tokens=False)) for cand in x['candidates']] for x in agg_data]
    cand_lens = list(chain(*cand_lens))
    ref_lens = [len(tokenizer.encode(x['output'], add_special_tokens=False))  for x in agg_data]
    logging.info("Input length statistics:")
    logging.info("  Avg/Max/90%/50%: {}/{}/{}/{}".format(np.mean(input_lens), np.max(input_lens), np.percentile(input_lens, 90), np.percentile(input_lens, 50)))
    logging.info("Candidate length statistics:")
    logging.info("  Avg/Max/90%: {}/{}/{}".format(np.mean(cand_lens), np.max(cand_lens), np.percentile(cand_lens, 90)))
    logging.info("Reference length statistics:")
    logging.info("  Avg/Max/90%: {}/{}/{}".format(np.mean(ref_lens), np.max(ref_lens), np.percentile(ref_lens, 90)))
    
    output_file = args.data_dir /"ins" / "train_data_new.json"
    with open(output_file, "w") as f:
        json.dump(agg_data, f, indent=4, ensure_ascii=False)
        logging.info("Saved aggregated data to {}".format(output_file))

tigerscore/prepare_data/aggregate_ins_train_data.py
TIGER-AI-Lab/TIGERScore
import json
import argparse
import logging
import numpy as np
from pathlib import Path
from collections import Counter
from itertools import chain
from transformers import AutoTokenizer
logging.basicConfig(level=logging.INFO)
"""
Datasets:
"""
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='../../data')
    args = parser.parse_args()
    args.data_dir = Path(args.data_dir)
    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
    agg_data = []
    datasets = ["din0s/asqa","DongfuTingle/FeTaQA","cosmos_qa","eli5"]
    logging.info("Aggregating data from {}".format(args.data_dir))
    np.random.seed(42)
    for dataset in datasets:
        with open(args.data_dir /"{}".format(dataset) /"train_data_prepared.json", "r") as f:
            data = json.load(f)
            data = [x for x in data if len(x['candidates']) > 0]
            data = [x for x in data if len(tokenizer.encode(x['input'], add_special_tokens=False)) < 600]
            np.random.shuffle(data)
            # give candidates
            # data = list(np.random.choice(data, 3000, replace=False))
            # per 5
            # data = list(np.random.choice(data, len(data)//3*2,replace=False))
            for i, item in enumerate(data):
                new_cands = sorted(item['candidates'], key=lambda x: x['scores']['bart_score'], reverse=True)
                if i < len(data)//3 :
                    item['candidates'] = [new_cands[0]]
                elif i < len(data)//3 * 2:
                    item['candidates'] = [new_cands[2]]
                else:
                    item['candidates'] = [new_cands[-1]]
            np.random.shuffle(data)
            # data = list(np.random.choice(data, 2000, replace=False))
            # data = list(np.random.choice(data, len(data)//2,replace=False))
            logging.info("Loaded {} data from {}".format(len(data), args.data_dir /"{}".format(dataset) /"train_data_prepared.json"))
            logging.info("Outputs: {}".format(np.sum([len(x['candidates']) for x in data])))
            for x in data:
                x['dataset'] = dataset
            agg_data += data
    logging.info("# Total aggregated # {} data for data to text".format(len(agg_data)))
    logging.info("# Outputs: {}".format(np.sum([len(x['candidates']) for x in agg_data])))
    # length statistics
    input_lens = [len(tokenizer.encode(x['input'], add_special_tokens=False)) for x in agg_data]
    cand_lens = [[len(tokenizer.encode(cand['text'], add_special_tokens=False)) for cand in x['candidates']] for x in agg_data]
    cand_lens = list(chain(*cand_lens))
    if dataset == "webnlg":
        ref_lens = [[len(tokenizer.encode(ref, add_special_tokens=False)) for ref in x['output']] for x in agg_data]
        ref_lens = list(chain(*ref_lens))
    else:
        ref_lens = [len(tokenizer.encode(x['output'], add_special_tokens=False))  for x in agg_data]
    logging.info("Input length statistics:")
    logging.info("  Avg/Max/90%/50%: {}/{}/{}/{}".format(np.mean(input_lens), np.max(input_lens), np.percentile(input_lens, 90), np.percentile(input_lens, 50)))
    logging.info("Candidate length statistics:")
    logging.info("  Avg/Max/90%: {}/{}/{}".format(np.mean(cand_lens), np.max(cand_lens), np.percentile(cand_lens, 90)))
    logging.info("Reference length statistics:")
    logging.info("  Avg/Max/90%: {}/{}/{}".format(np.mean(ref_lens), np.max(ref_lens), np.percentile(ref_lens, 90)))
    
    output_file = args.data_dir /"lfqa" / "train_data_new.json"
    with open(output_file, "w") as f:
        json.dump(agg_data, f, indent=4, ensure_ascii=False)
        logging.info("Saved aggregated data to {}".format(output_file))







tigerscore/prepare_data/aggregate_lfqa_train_data.py
TIGER-AI-Lab/TIGERScore
import json
import argparse
import logging
import numpy as np
from pathlib import Path
from collections import Counter
from itertools import chain
from transformers import AutoTokenizer
logging.basicConfig(level=logging.INFO)
"""
Datasets:
    - SummEval(CNNDM)
    - XSUM
    - Newsroom
    - Samsum
"""
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='../../data')
    args = parser.parse_args()
    args.data_dir = Path(args.data_dir)
    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
    agg_data = []
    datasets = ["summeval","xsum","newsroom","samsum"]
    np.random.seed(42)
    logging.info("Aggregating data from {}".format(args.data_dir))
    for dataset in datasets:
        with open(args.data_dir /"{}".format(dataset) /"train_data_prepared.json", "r") as f:
            data = json.load(f)
            data = [x for x in data if len(x['candidates']) > 0]
            # 放宽长度，提升Unique outputs，均匀的error数量，均匀的candidate质量
            data = [x for x in data if len(tokenizer.encode(x['input'], add_special_tokens=False)) < 800 and  len(tokenizer.encode(x['output'], add_special_tokens=False)) < 400]
            np.random.shuffle(data)
            if dataset != "summeval":
                for i, item in enumerate(data):
                    new_cands = sorted(item['candidates'], key=lambda x: x['scores']['bart_score_cnn'], reverse=True)
                    if i < len(data)//6 :
                        item['candidates'] = [new_cands[0]]
                    elif i < len(data)//3:
                        item['candidates'] = [new_cands[2]]
                    else:
                        item['candidates'] = [new_cands[-1]]
            else:
                for i, item in enumerate(data):
                    new_cands = sorted(item['candidates'], key=lambda x: x['scores']["bart_score_cnn"], reverse=True)
                    item['candidates'] = [new_cands[0],new_cands[len(new_cands)//2],new_cands[-1]]
            logging.info("Loaded {} data from {}".format(len(data), args.data_dir /"{}".format(dataset) /"train_data_prepared.json"))
            logging.info("  Unique outputs: {}".format(np.sum([len(x['candidates']) for x in data])))
            for x in data:
                x['dataset'] = dataset
            agg_data += data
    logging.info("# Total aggregated # {} data for summarization".format(len(agg_data)))
    logging.info("# Unique outputs: {}".format(np.sum([len(x['candidates']) for x in agg_data])))
    # length statistics
    input_lens = [len(tokenizer.encode(x['input'], add_special_tokens=False)) for x in agg_data]
    cand_lens = [[len(tokenizer.encode(cand['text'], add_special_tokens=False)) for cand in x['candidates']] for x in agg_data]
    cand_lens = list(chain(*cand_lens))
    ref_lens = [len(tokenizer.encode(x['output'], add_special_tokens=False))  for x in agg_data]
    logging.info("Input length statistics:")
    logging.info("  Avg/Max/90%/50%: {}/{}/{}/{}".format(np.mean(input_lens), np.max(input_lens), np.percentile(input_lens, 90), np.percentile(input_lens, 50)))
    logging.info("Candidate length statistics:")
    logging.info("  Avg/Max/90%: {}/{}/{}".format(np.mean(cand_lens), np.max(cand_lens), np.percentile(cand_lens, 90)))
    logging.info("Reference length statistics:")
    logging.info("  Avg/Max/90%: {}/{}/{}".format(np.mean(ref_lens), np.max(ref_lens), np.percentile(ref_lens, 90)))
    output_file = args.data_dir /"sum" / "train_data_new.json"
    with open(output_file, "w") as f:
        json.dump(agg_data, f, indent=4, ensure_ascii=False)
        logging.info("Saved aggregated data to {}".format(output_file))







tigerscore/prepare_data/aggregate_sum_train_data.py
TIGER-AI-Lab/TIGERScore
import json
import argparse
import logging
import numpy as np
from pathlib import Path
from collections import Counter
from itertools import chain
from transformers import AutoTokenizer
logging.basicConfig(level=logging.INFO)

lang_map = {
    'zh': 'Chinese',
    'en': 'English',
    'de': 'German',
    'ru': 'Russian',
    'cs': 'Czech',
    'uk': 'Ukrainian',
    'hr': 'Croatian',
    'ja': 'Japanese',
    'liv': 'Livonian',
    'fi': 'Finnish',
    'fr': 'French',
}

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='../../data/wmt')
    args = parser.parse_args()
    args.data_dir = Path(args.data_dir)
    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")

    agg_data = []
    logging.info("Aggregating data from {}".format(args.data_dir))
    for lang_pair in args.data_dir.iterdir():
        if not lang_pair.is_dir():
            continue
        src_lang, tgt_lang = lang_pair.name.split("-")
        with open(lang_pair / "train_data.json") as f:
            data = json.load(f)
        agg_data.extend(data)
    logging.info("# Total aggregated # {} data for translation".format(len(agg_data)))
    logging.info("# Unique outputs: {}".format(np.sum([len(x['candidates']) for x in agg_data])))
    # report total statistics
    # year statistics
    logging.info("Year statistics:")
    year_counter = Counter([x['data_source'].split("_")[0] for x in agg_data])
    for year in year_counter:
        logging.info("  {}: {}".format(year, year_counter[year]))
    # lang_pair statistics
    logging.info("Lang pair statistics:")
    lang_pair_counter = Counter([x['data_source'].split("_")[1] for x in agg_data])
    for lang_pair in lang_pair_counter:
        logging.info("  {}: {}".format(lang_pair, lang_pair_counter[lang_pair]))
    # domain statistics
    logging.info("Domain statistics:")
    domain_counter = Counter([x['data_source'].split("_")[-1] for x in agg_data])
    for domain in domain_counter:
        logging.info("  {}: {}".format(domain, domain_counter[domain]))
    # Model statistics
    logging.info("System models statistics:")
    model_counter = Counter(list(chain(*[[cand['model'] for cand in x['candidates']] for x in agg_data])))
    for model in model_counter:
        logging.info("  {}: {}".format(model, model_counter[model]))
    # length statistics
    input_lens = [len(tokenizer.encode(x['input'], add_special_tokens=False)) for x in agg_data]
    cand_lens = [[len(tokenizer.encode(cand['text'], add_special_tokens=False)) for cand in x['candidates']] for x in agg_data]
    cand_lens = list(chain(*cand_lens))
    ref_lens = [[len(tokenizer.encode(ref, add_special_tokens=False)) for ref in x['refs']] for x in agg_data]
    ref_lens = list(chain(*ref_lens))
    logging.info("Input length statistics:")
    logging.info("  Min/Avg/Max/90%: {}/{}/{}/{}".format(np.min(input_lens), np.mean(input_lens), np.max(input_lens), np.percentile(input_lens, 90)))
    logging.info("Candidate length statistics:")
    logging.info("  Min/Avg/Max/90%: {}/{}/{}/{}".format(np.min(cand_lens), np.mean(cand_lens), np.max(cand_lens), np.percentile(cand_lens, 90)))
    logging.info("Reference length statistics:")
    logging.info("  Min/Avg/Max/90%: {}/{}/{}/{}".format(np.min(ref_lens), np.mean(ref_lens), np.max(ref_lens), np.percentile(ref_lens, 90)))
    with open(args.data_dir / "train_data.json", "w") as f:
        json.dump(agg_data, f, indent=4, ensure_ascii=False)
        logging.info("Saved aggregated data to {}".format(args.data_dir / "train_data.json"))

tigerscore/prepare_data/aggregate_wmt_train_data.py
TIGER-AI-Lab/TIGERScore
import sys
import os
from pathlib import Path
import json
sys.path.append("/home//WorkSpace/ExplainableGPTScore_bak/src")
from common.evaluation import overall_eval

output_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/summeval/train_data_prepared.json"

with open(output_file, "r") as f:
    datas = json.load(f)
candidates = []
targets = []
candidates = [[candidate["text"] for candidate in data["candidates"] ]for data in datas]
targets = [data["output"] for data in datas]
print(candidates[0], targets[0])
metrics = ["bleu", "rouge1","rouge2","rougeL","bart_score","bart_score_cnn"]
# DS = datas
# DS = {x['id']: x for x in DS}
# pure_candidates = [[x['text'] for x in item['candidates']] for item in candidates]
# targets = [DS[x['id']]['output'] for x in candidates]
# evaluate
scores = overall_eval(candidates, targets, metrics, 1)

# scores = overall_eval( candidates,targets, metrics=metrics)

for metric in metrics:
    scores_metric = scores[metric]
    for i, sample in enumerate(datas):
        _sample_cands = sample['candidates']
        for j, _sample_cand in enumerate(_sample_cands):
            _sample_cand['scores'][metric] = scores_metric[i][j]
            
with open(output_file, "w") as f:
    json.dump(datas, f, indent=4)
tigerscore/prepare_data/auto_metrics_eval.py
TIGER-AI-Lab/TIGERScore
import json
from collections import defaultdict
from matplotlib import pyplot as plt
import numpy as np
task = "data2text"
task_data_map = {
    "data2text": "/home//WorkSpace/ExplainableGPTScore_bak/data/d2t/train_data.align_score.filter_v1.json"
}
with open(task_data_map[task], 'r') as f:
    data = json.load(f)

import sys
sys.path.append("/home//WorkSpace/ExplainableGPTScore_bak/src")
from xgptscore.constants import EVAL_ASPECTS
xgptscore_types = ['xgptscore']
xgptscore_types = xgptscore_types + list(EVAL_ASPECTS[task].keys())
# xgptscore_types.remove('Coherence')
print(xgptscore_types)
scores = defaultdict(list)
datasets = defaultdict(int)
datasets_can = defaultdict(int)
for item in data:
    datasets[item['dataset']] += 1
    for cand in item['candidates']:
        datasets_can[item['dataset']] += 1
        cand_scores = defaultdict(int)
        if isinstance(cand['responses'][-1], dict):
            for error in cand['responses'][-1]['errors'].values():
                if error['score_reduction']: # is Not None and is not 0
                    cand_scores[error['error_aspect']] -= error['score_reduction']
                    cand_scores['xgptscore'] -= error['score_reduction']
        for key in xgptscore_types:
            scores[key].append(cand_scores[key])

        
fig, axes = plt.subplots(len(xgptscore_types) // 2 + 1, 2)
for i, key in enumerate(scores):
    ax = axes[i // 2, i % 2]
    # make bins' width wider 
    ax.hist(scores[key], bins=40)
    ax.set_title(key)

plt.tight_layout()
plt.show()
print(datasets)
print(datasets_can)
print(len([x for x in scores['xgptscore']]))
print(len([x for x in scores['xgptscore'] if x == 0]) / len(scores['xgptscore']))

#filter
from fuzzywuzzy import fuzz
from copy import deepcopy
new_data = []
tmp_data = deepcopy(data)
# merge coherence and fluency
ratios = defaultdict(list)
ratios_75 = {
    'totto': 47,
    'kasnerz/wikitabletext': 42,
    'webnlg': 21,
}
ratios_50 = {
    'totto': 68.0,
    'kasnerz/wikitabletext': 73,
    'webnlg': 35,
}
for item in tmp_data:
    new_cands = []
    for cand in item['candidates']:
        put_in = True
        cand_scores = defaultdict(int)
        if isinstance(cand['responses'][-1], dict):
            for error in cand['responses'][-1]['errors'].values():
                if not isinstance(error, dict):
                    put_in = False
                for key in error.keys():
                    if key not in ["error_aspect", "severity", "explanation", "error_location", "score_reduction"]:
                        put_in = False
                        # print(key)
                for value in error.values():
                    if value is None:
                        put_in = False

                if error["error_aspect"] in ["Coherence", "Fluency", "Coherence, Fluency"]:
                    error["error_aspect"] = "Fluency"

                if error['score_reduction'] is None or error['score_reduction'] < 0:
                    put_in = False
                try:
                    float(error['score_reduction'])
                except:
                    put_in = False
                
                if error["error_aspect"] not in EVAL_ASPECTS[task]:
                    put_in = False
                    # print(error["error_aspect"])
                if error["severity"] not in ["Major", "Minor"]:
                    put_in = False
                    # print(error["severity"])
                if error["explanation"] in ["", " ","N/A","None"]:
                    put_in = False
                    # print(error["explanation"])
                
                # if error["error_location"] is not None and error["error_location"] not in item["input"]:
                #     put_in = False
                #     # print(error["error_location"])
                # ratios[item["dataset"]].append(fuzz.token_set_ratio(item["input"], error["error_location"]))
                if fuzz.token_set_ratio(item["input"], error["error_location"]) < ratios_75[item["dataset"]]:
                    put_in = False
        else:
            put_in = False
        if put_in:
            new_cands.append(cand)
    if len(new_cands) > 0:
        item['candidates'] = new_cands
        new_data.append(item)
    
print(f"Old data: {len(data)}")
print(f"Old data Candidates: {sum([len(item['candidates']) for item in data])}")
print(f"New data: {len(new_data)}")
print(f"New data Candidates: {sum([len(item['candidates']) for item in new_data])}")

new_output_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/d2t/train_data.align_score.filter_v1.json"
with open(new_output_file, 'w') as f:
    json.dump(new_data, f, indent=4)
tigerscore/prepare_data/filter_d2t_distill_data.py
TIGER-AI-Lab/TIGERScore
# %%
import json
from collections import defaultdict
from matplotlib import pyplot as plt
import numpy as np
from collections import Counter
task = "instruction-following"
task_data_map = {
    "instruction-following": "/home//WorkSpace/ExplainableGPTScore_bak/data/ins/train_data_new.instruction_following.v3.json"
}
with open(task_data_map[task], 'r') as f:
    data = json.load(f)
# all_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/sum/train_data_all.json"
# with open(all_file, 'r') as f:
#     all_data = json.load(f)
# for item in data:
#     for _item in all_data:
#         if item["input"] == _item["input"]:
#             item["dataset"] = _item["dataset"]
#             break
# with open(task_data_map[task], 'w') as f:
#     json.dump(data, f, indent=4)

# %%
import sys
sys.path.append("/home//WorkSpace/ExplainableGPTScore_bak/src")
from xgptscore.constants import EVAL_ASPECTS
xgptscore_types = ['xgptscore',"Comprehension",'Accuracy',"Informativeness","Coherence"]
# xgptscore_types = xgptscore_types + list(EVAL_ASPECTS[task].keys())
# xgptscore_types.remove('Coherence')
print(xgptscore_types)
scores = defaultdict(list)
datasets = defaultdict(int)
datasets_can = defaultdict(int)
penlty_distribution = defaultdict(int)
penlty_distribution_per_dataset = defaultdict(lambda: defaultdict(int))
score_distribution = defaultdict(int)
score_distribution_per_aspect = defaultdict(lambda: defaultdict(int))
for item in data:
    # datasets[item['dataset']] += 1
    for cand in item['candidates']:
        # datasets_can[item['dataset']] += 1
        cand_scores = defaultdict(int)
        penlty_num = 0
        if isinstance(cand['responses'][-1], dict):
            for error in cand['responses'][-1]['errors'].values():
                if error['score_reduction']: # is Not None and is not 0
                    cand_scores[error['error_aspect']] -= error['score_reduction']
                    cand_scores['xgptscore'] -= error['score_reduction']
                    penlty_num += 1
                    score_distribution[error['score_reduction']] += 1
                    score_distribution_per_aspect[error['error_aspect']][error['score_reduction']] += 1
        for key in xgptscore_types:
            scores[key].append(cand_scores[key])
        penlty_distribution[penlty_num] += 1
        # penlty_distribution_per_dataset[item['dataset']][penlty_num] += 1
        

        
fig, axes = plt.subplots(len(xgptscore_types) // 2 + 1, 2)
for i, key in enumerate(scores):
    ax = axes[i // 2, i % 2]
    # make bins' width wider 
    ax.hist(scores[key], bins=np.arange(min(scores[key]), max(scores[key]) + 1, 1))
    print(min(scores[key]), max(scores[key]))
    ax.set_title(key)

plt.tight_layout()
plt.show()
# print(datasets)
# print(datasets_can)
# print(penlty_distribution_per_dataset)
print("score distribution:",Counter(score_distribution))
for key in score_distribution_per_aspect:
    print(key,":",Counter(score_distribution_per_aspect[key]))
print(Counter([x for x in scores['xgptscore']]))
print("penlty distribution",Counter(penlty_distribution))
# print("penlty distribution",penlty_distribution_per_dataset)
print("total candidates",len([x for x in scores['xgptscore']]))
print("0 ratio",len([x for x in scores['xgptscore'] if x == 0]) / len(scores['xgptscore']))

# %%
print(data[0])

# %%
#filter
from fuzzywuzzy import fuzz
from copy import deepcopy
new_data = []
tmp_data = deepcopy(data)
# merge coherence and fluency
ratios = []
# ratios_75 = {
#     'totto': 47,
#     'kasnerz/wikitabletext': 42,
#     'webnlg': 21,
# }
# ratios_50 = {
#     'totto': 68.0,
#     'kasnerz/wikitabletext': 73,
#     'webnlg': 35,
# }
need_to_remove = 0
for item in tmp_data:
    new_cands = []
    for cand in item['candidates']:
        put_in = True
        cand_scores = defaultdict(int)
        if isinstance(cand['responses'][-1], dict):
            if len(cand['responses'][-1]['errors'].values()) > 10:
                put_in = False
            for error in cand['responses'][-1]['errors'].values():
                if not isinstance(error, dict):
                    put_in = False
                for key in error.keys():
                    if key not in ["error_aspect", "severity", "explanation", "error_location", "score_reduction"]:
                        put_in = False
                        # print(key)
                for value in error.values():
                    if value is None:
                        put_in = False
                if error['score_reduction'] is None or error['score_reduction'] < 0:
                    put_in = False
                try:
                    float(error['score_reduction'])
                except:
                    put_in = False
                
                if error["error_aspect"] not in ["Comprehension",'Accuracy',"Informativeness","Coherence"]:
                    put_in = False
                    # print(error["error_aspect"])
                if error["severity"] not in ["Major", "Minor"]:
                    put_in = False
                    # print(error["severity"])
                if error["explanation"] in ["", " ","N/A","None"]:
                    put_in = False
                    # print(error["explanation"])
                if error["score_reduction"] > 5:
                    put_in = False
                if error["severity"] == "Major" and error["score_reduction"] < 2:
                    put_in = False
                # # if error["error_location"] is not None and error["error_location"] not in cand["text"]:
                #     put_in = False
                #     # print(error["error_location"])
                # if error["score_reduction"] <= 1:
                #     # ratios[item["dataset"]].append(fuzz.token_set_ratio(cand["text"], error["error_location"]))
                #     ratios[item["dataset"]].append(fuzz.o(cand["text"], error["explanation"]))
                    # if fuzz.token_set_ratio(cand["text"], error["error_location"]) < 100:
                    #     put_in = False
                #     put_in = False
                #     # print(error["error_location"])
                # if fuzz.token_set_ratio(cand["text"], error["error_location"]) < 50:
                #     put_in = False
                ratios.append(fuzz.token_set_ratio(cand["text"], error["error_location"]))
            # if len(cand['responses'][-1]['errors'].values()) in [1,2]:
            # #     error = list(cand['responses'][-1]['errors'].values())[0]
            # #     if error["error_aspect"] == "Accuracy" and error["severity"] == "Minor":、
            #     for error in cand['responses'][-1]['errors'].values():
            #         ratios[item["dataset"]].append(fuzz.token_set_ratio(cand["text"], error["error_location"]))
                # if not error["error_aspect"] == "Fluency":
                    # if fuzz.token_set_ratio(cand["text"], error["error_location"]) < 100:
                    #     put_in = False
        else:
            put_in = False
        if put_in:
            new_cands.append(cand)
    if len(new_cands) > 0:
        item['candidates'] = new_cands
        new_data.append(item)
    
print(f"Old data: {len(data)}")
print(f"Old data Candidates: {sum([len(item['candidates']) for item in data])}")
print(f"New data: {len(new_data)}")
print(f"New data Candidates: {sum([len(item['candidates']) for item in new_data])}")
print(need_to_remove)

new_output_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/ins/train_data_new.instruction_following.clean.json"
with open(new_output_file, 'w') as f:
    json.dump(new_data, f, indent=4)

# %%
from copy import deepcopy
import numpy as np
new_data = []
tmp_data = deepcopy(data)
np.random.seed(42)


for item in tmp_data:
    new_cands = []
    for cand in item['candidates']:
        need_to_remove = False
        for error in cand['responses'][-1]['errors'].values():
            if error["error_aspect"] != "Fluency":
                if fuzz.token_set_ratio(cand["text"], error["error_location"]) < 50:
                    need_to_remove = True
        if cand['scores']['xgptscore'] == 0 and np.random.random() < 1/2:
            continue
        # if len(cand['responses'][-1]['errors'].values()) == 1:
        #     error = list(cand['responses'][-1]['errors'].values())[0]
        #     if error["error_aspect"] != "Fluency" and error["score_reduction"] <= 1 and np.random.random() < 1/2:
        #         continue
        #     # if error["score_reduction"] <= 1:
        #     #     if fuzz.token_set_ratio(cand["text"], error["error_location"]) < 100:
        #     #         continue
        
        if not need_to_remove:
            new_cands.append(cand)
    if len(new_cands) == 0:
        continue
    item['candidates'] = new_cands
    new_data.append(item)
            
    
    
print(f"Old data: {len(data)}")
print(f"Old data Candidates: {sum([len(item['candidates']) for item in data])}")
print(f"New data: {len(new_data)}")
print(f"New data Candidates: {sum([len(item['candidates']) for item in new_data])}")

new_output_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/ins/train_data_new.instruction_following.v3.json"
with open(new_output_file, 'w') as f:
    json.dump(new_data, f, indent=4)

# %%
import numpy as np



print(np.max(ratios),np.percentile(ratios,90),np.percentile(ratios,75),np.percentile(ratios,50),np.percentile(ratios,25),np.percentile(ratios,10),np.min(ratios))



tigerscore/prepare_data/filter_ins_distill_data.py
TIGER-AI-Lab/TIGERScore
# %%
import json
from collections import defaultdict
from matplotlib import pyplot as plt
import numpy as np
from collections import Counter
task = "long-form QA"
task_data_map = {
    "long-form QA": "/home//WorkSpace/ExplainableGPTScore_bak/data/lfqa/train_data_new.longform_qa.v3.json"
}
with open(task_data_map[task], 'r') as f:
    data = json.load(f)
# all_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/sum/train_data_all.json"
# with open(all_file, 'r') as f:
#     all_data = json.load(f)
# for item in data:
#     for _item in all_data:
#         if item["input"] == _item["input"]:
#             item["dataset"] = _item["dataset"]
#             break
# with open(task_data_map[task], 'w') as f:
#     json.dump(data, f, indent=4)

# %%
import sys
sys.path.append("/home//WorkSpace/ExplainableGPTScore_bak/src")
from xgptscore.constants import EVAL_ASPECTS
xgptscore_types = ['xgptscore',"Clarity",'Accuracy',"Fluency"]
# xgptscore_types = xgptscore_types + list(EVAL_ASPECTS[task].keys())
# xgptscore_types.remove('Coherence')
print(xgptscore_types)
scores = defaultdict(list)
datasets = defaultdict(int)
datasets_can = defaultdict(int)
penlty_distribution = defaultdict(int)
penlty_distribution_per_dataset = defaultdict(lambda: defaultdict(int))
score_distribution = defaultdict(int)
score_distribution_per_aspect = defaultdict(lambda: defaultdict(int))
for item in data:
    datasets[item['dataset']] += 1
    for cand in item['candidates']:
        datasets_can[item['dataset']] += 1
        cand_scores = defaultdict(int)
        penlty_num = 0
        if isinstance(cand['responses'][-1], dict):
            for error in cand['responses'][-1]['errors'].values():
                if error['score_reduction']: # is Not None and is not 0
                    cand_scores[error['error_aspect']] -= error['score_reduction']
                    cand_scores['xgptscore'] -= error['score_reduction']
                    penlty_num += 1
                    score_distribution[error['score_reduction']] += 1
                    score_distribution_per_aspect[error['error_aspect']][error['score_reduction']] += 1
        for key in xgptscore_types:
            scores[key].append(cand_scores[key])
        penlty_distribution[penlty_num] += 1
        penlty_distribution_per_dataset[item['dataset']][penlty_num] += 1
        

        
fig, axes = plt.subplots(len(xgptscore_types) // 2 + 1, 2)
for i, key in enumerate(scores):
    ax = axes[i // 2, i % 2]
    # make bins' width wider 
    ax.hist(scores[key], bins=np.arange(min(scores[key]), max(scores[key]) + 1, 1))
    print(min(scores[key]), max(scores[key]))
    ax.set_title(key)

plt.tight_layout()
plt.show()
print(datasets)
print(datasets_can)
# print(penlty_distribution_per_dataset)
print("score distribution:",Counter(score_distribution))
for key in score_distribution_per_aspect:
    print(key,":",Counter(score_distribution_per_aspect[key]))
print(Counter([x for x in scores['xgptscore']]))
print("penlty distribution",Counter(penlty_distribution))
# print("penlty distribution",penlty_distribution_per_dataset)
print("total candidates",len([x for x in scores['xgptscore']]))
print("0 ratio",len([x for x in scores['xgptscore'] if x == 0]) / len(scores['xgptscore']))

# %%
print(data[0])

# %%
#filter
from fuzzywuzzy import fuzz
from copy import deepcopy
new_data = []
tmp_data = deepcopy(data)
# merge coherence and fluency
ratios = defaultdict(list)
# ratios_75 = {
#     'totto': 47,
#     'kasnerz/wikitabletext': 42,
#     'webnlg': 21,
# }
# ratios_50 = {
#     'totto': 68.0,
#     'kasnerz/wikitabletext': 73,
#     'webnlg': 35,
# }
need_to_remove = 0
for item in tmp_data:
    new_cands = []
    for cand in item['candidates']:
        put_in = True
        cand_scores = defaultdict(int)
        if isinstance(cand['responses'][-1], dict):
            if len(cand['responses'][-1]['errors'].values()) > 10:
                put_in = False
            for error in cand['responses'][-1]['errors'].values():
                if not isinstance(error, dict):
                    put_in = False
                for key in error.keys():
                    if key not in ["error_aspect", "severity", "explanation", "error_location", "score_reduction"]:
                        put_in = False
                        # print(key)
                for value in error.values():
                    if value is None:
                        put_in = False

                if error["error_aspect"] in ["Coherence", "Fluency", "Coherence, Fluency"]:
                    error["error_aspect"] = "Fluency"

                if error['score_reduction'] is None or error['score_reduction'] < 0:
                    put_in = False
                try:
                    float(error['score_reduction'])
                except:
                    put_in = False
                
                if error["error_aspect"] not in ["Clarity",'Accuracy',"Fluency"]:
                    put_in = False
                    # print(error["error_aspect"])
                if error["severity"] not in ["Major", "Minor"]:
                    put_in = False
                    # print(error["severity"])
                if error["explanation"] in ["", " ","N/A","None"]:
                    put_in = False
                    # print(error["explanation"])
                if error["score_reduction"] > 5:
                    put_in = False
                    # print(error["score_reduction"])
                if error["severity"] == "Major" and error["score_reduction"] < 2:
                    put_in = False
                # # if error["error_location"] is not None and error["error_location"] not in cand["text"]:
                #     put_in = False
                #     # print(error["error_location"])
                # if error["score_reduction"] <= 1:
                #     # ratios[item["dataset"]].append(fuzz.token_set_ratio(cand["text"], error["error_location"]))
                #     ratios[item["dataset"]].append(fuzz.o(cand["text"], error["explanation"]))
                    # if fuzz.token_set_ratio(cand["text"], error["error_location"]) < 100:
                    #     put_in = False
                #     put_in = False
                #     # print(error["error_location"])
                # if fuzz.token_set_ratio(cand["text"], error["error_location"]) < 50:
                #     put_in = False
                if error["severity"] == "Minor":
                    ratios[item["dataset"]].append(fuzz.token_set_ratio(cand["text"], error["error_location"]))
            # if len(cand['responses'][-1]['errors'].values()) in [1,2]:
            # #     error = list(cand['responses'][-1]['errors'].values())[0]
            # #     if error["error_aspect"] == "Accuracy" and error["severity"] == "Minor":、
            #     for error in cand['responses'][-1]['errors'].values():
            #         ratios[item["dataset"]].append(fuzz.token_set_ratio(cand["text"], error["error_location"]))
                # if not error["error_aspect"] == "Fluency":
                    # if fuzz.token_set_ratio(cand["text"], error["error_location"]) < 100:
                    #     put_in = False
        else:
            put_in = False
        if put_in:
            new_cands.append(cand)
    if len(new_cands) > 0:
        item['candidates'] = new_cands
        new_data.append(item)
    
print(f"Old data: {len(data)}")
print(f"Old data Candidates: {sum([len(item['candidates']) for item in data])}")
print(f"New data: {len(new_data)}")
print(f"New data Candidates: {sum([len(item['candidates']) for item in new_data])}")
print(need_to_remove)

# new_output_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/lfqa/train_data_new.longform_qa.clean.json"
# with open(new_output_file, 'w') as f:
#     json.dump(new_data, f, indent=4)

# %%
from copy import deepcopy
import numpy as np
new_data = []
tmp_data = deepcopy(data)
np.random.seed(42)

# ratios_50 = {
#     'totto': 80.0,
#     'kasnerz/wikitabletext': 84,
#     'webnlg': 43,
# }


ratios_50 = {
    'din0s/asqa': 85,
    'Tingle/FeTaQA': 85,
    'cosmos_qa': 75,
    'eli5': 50,
}

for item in tmp_data:
    new_cands = []
    for cand in item['candidates']:
        need_to_remove = False
        for error in cand['responses'][-1]['errors'].values():
            if error["error_aspect"] != "Fluency":
                if fuzz.token_set_ratio(cand["text"], error["error_location"]) < ratios_50[item["dataset"]]:
                    need_to_remove = True
        if cand['scores']['xgptscore'] == 0 and np.random.random() < 1/2:
            continue
        # if len(cand['responses'][-1]['errors'].values()) == 1:
        #     error = list(cand['responses'][-1]['errors'].values())[0]
        #     if error["error_aspect"] != "Fluency" and error["score_reduction"] <= 1 and np.random.random() < 1/2:
        #         continue
        #     # if error["score_reduction"] <= 1:
        #     #     if fuzz.token_set_ratio(cand["text"], error["error_location"]) < 100:
        #     #         continue
        
        if not need_to_remove:
            new_cands.append(cand)
    if len(new_cands) == 0:
        continue
    item['candidates'] = new_cands
    new_data.append(item)
            
    
    
print(f"Old data: {len(data)}")
print(f"Old data Candidates: {sum([len(item['candidates']) for item in data])}")
print(f"New data: {len(new_data)}")
print(f"New data Candidates: {sum([len(item['candidates']) for item in new_data])}")

new_output_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/lfqa/train_data_new.longform_qa.v3.json"
with open(new_output_file, 'w') as f:
    json.dump(new_data, f, indent=4)

# %%
import numpy as np


for key,value in ratios.items():
    print(key,len(value))
    print(key,np.max(value),np.percentile(value,90),np.percentile(value,75),np.percentile(value,50),np.percentile(value,25),np.percentile(value,10),np.min(value))



tigerscore/prepare_data/filter_lfqa_distill_data.py
TIGER-AI-Lab/TIGERScore
import json
from collections import defaultdict
from matplotlib import pyplot as plt
import numpy as np
task = "summarization"
task_data_map = {
    "summarization": "/home//WorkSpace/ExplainableGPTScore_bak/data/sum/train_data.align_score.distill.json"
}
with open(task_data_map[task], 'r') as f:
    data = json.load(f)

#filter
from fuzzywuzzy import fuzz
from copy import deepcopy
new_data = []
tmp_data = deepcopy(data)
# merge coherence and fluency
ratios = {
    'summeval': [],
    'xsum': [],
    'newsroom': [],
    'samsum': [],
}
ratios_75 = {
    'summeval': 64,
    'xsum': 40,
    'newsroom': 45,
    'samsum': 48,
} # get from numpy
ratios_50 = {
    'summeval': 78,
    'xsum': 62,
    'newsroom': 64,
    'samsum': 70,
}
for item in tmp_data:
    new_cands = []
    for cand in item['candidates']:
        put_in = True
        cand_scores = defaultdict(int)
        if isinstance(cand['responses'][-1], dict):
            for error in cand['responses'][-1]['errors'].values():
                if not isinstance(error, dict):
                    put_in = False
                for key in error.keys():
                    if key not in ["error_aspect", "severity", "explanation", "error_location", "score_reduction"]:
                        put_in = False
                        # print(key)
                for value in error.values():
                    if value is None:
                        put_in = False

                if error["error_aspect"] in ["Coherence", "Fluency", "Coherence, Fluency"]:
                    error["error_aspect"] = "Fluency"

                if error['score_reduction'] is None or error['score_reduction'] < 0:
                    put_in = False
                try:
                    float(error['score_reduction'])
                except:
                    put_in = False
                
                if error["error_aspect"] not in EVAL_ASPECTS["summarization"]:
                    put_in = False
                    # print(error["error_aspect"])
                if error["severity"] not in ["Major", "Minor"]:
                    put_in = False
                    # print(error["severity"])
                if error["explanation"] in ["", " ","N/A","None"]:
                    put_in = False
                    # print(error["explanation"])
                
                # if error["error_location"] is not None and error["error_location"] not in item["input"]:
                #     put_in = False
                #     # print(error["error_location"])

                
                # ratios[item["dataset"]].append(fuzz.token_set_ratio(item["input"], error["error_location"]))
                if fuzz.token_set_ratio(item["input"], error["error_location"]) < ratios_50[item["dataset"]]:
                    put_in = False
        else:
            put_in = False
        if put_in:
            new_cands.append(cand)
    if len(new_cands) > 0:
        item['candidates'] = new_cands
        new_data.append(item)

import numpy as np

for key,value in ratios.items():
    print(key,np.max(value),np.percentile(value,90),np.percentile(value,75),np.percentile(value,50),np.percentile(value,25),np.min(value))
    
print(f"Old data: {len(data)}")
print(f"Old data Candidates: {sum([len(item['candidates']) for item in data])}")
print(f"New data: {len(new_data)}")
print(f"New data Candidates: {sum([len(item['candidates']) for item in new_data])}")

new_output_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/sum/train_data.align_score.filter.json"
with open(new_output_file, 'w') as f:
    json.dump(new_data, f, indent=4)
tigerscore/prepare_data/filter_sum_distill_data.py
TIGER-AI-Lab/TIGERScore
"""
Get WebNLG2017 data
"""

"""
Get WMT train data
"""
import json
import argparse
import time
import logging
import numpy as np
from typing import List
from pathlib import Path
from datasets import load_dataset
from pathlib import Path
from collections import Counter
from itertools import chain
from transformers import AutoTokenizer

logging.basicConfig(level=logging.INFO)
# log current time
logging.info("\nRunning time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))

def format_eval_data(dataset, data_version):
    if data_version == 2017:
        return format_eval_data_2017(dataset)
    elif data_version == 2020:
        return format_train_data_2020(dataset)
    else:
        logging.error("Unknown data version: {}".format(data_version))
        return None

def format_eval_data_2017(dataset):
    dataset = sorted(dataset, key=lambda x: x['X_unit_id'])
    # merge data with the same id and get mean for human score
    merged_dataset = []
    
    now_id = dataset[0]['X_unit_id']
    now_items = []
    for item in dataset:
        x_unit_id = item['X_unit_id']
        if x_unit_id != now_id:
            # merge now_items
            now_items = merge_items(now_items)
            merged_dataset.append(now_items)
            now_items = []
            now_id = x_unit_id
            
    
    def merge_items(now_items: list) -> dict:
        res = now_items[0]
        scores ={
            "fluency": np.mean([item['fluency'] for item in now_items]),
            "grammaticality": np.mean([item['grammaticality'] for item in now_items]),
            "semantic_adequacy": np.mean([item['semantic_adequacy'] for item in now_items]),
        }
        res.update(scores)
        return res
    
    dataset = merged_dataset
    
    formated_data = {}
    src_id = 0
    for item in dataset:
        src = item['mr'].replace("|", ",").replace("<br>", "\n").strip()
        src = "\n".join(["( " + x.strip() + " )" for x in src.split("\n") if x.strip() != ""])
        if src not in formated_data:
            formated_data[src] = {
                "id": f"webnlg2017_human_eval_{src_id}",
                "instruction": "Generate a description for the following triples.",
                "input": src,
                "refs": [],
                "data_source": "webnlg2017",
                "task": "data2text",
                "candidates": [],
            }
            src_id += 1
        # add candidate and avoid duplicate candidates
        if any([cand['text'] == item['text'] for cand in formated_data[src]['candidates']]):
            continue
        if item['team'] == 'webnlg':
            formated_data[src]['refs'].append(item['text'])
            continue
        formated_data[src]['candidates'].append({
            "text": item['text'],
            "model": item['team'],
            "decoding_method": "greedy",
            "domain": item['category'],
            "type": item['type'],
            "scores": {
                "fluency": item['fluency'],
                "grammaticality": item['grammaticality'],
                "semantic_adequacy": item['semantic_adequacy'],
            }
        })
    train_data = list(formated_data.values())
    logging.info("Train data statistics:")
    logging.info("# Examples: {}".format(len(train_data)))
    logging.info("# Avg. Unique outputs: {}".format(sum([len(x['candidates']) for x in train_data]) / len(train_data)))
    logging.info("# Unique src: {}".format(len(set([x['input'] for x in train_data]))))
    logging.info("Domain distribution:")
    domain_counter = Counter([x['data_source'].split("_")[-1] for x in train_data])
    for domain in domain_counter:
        logging.info("  {}: {}".format(domain, domain_counter[domain]))
    logging.info("Type distribution:")
    type_counter = Counter([x['type'] for x in train_data])
    for type in type_counter:
        logging.info("  {}: {}".format(type, type_counter[type]))
    return train_data

def format_eval_data_2020(dataset):
    dataset = sorted(dataset, key=lambda x: x['X_unit_id'])
    # merge data with the same id and get mean for human score
    merged_dataset = []
    
    now_id = dataset[0]['X_unit_id']
    now_items = []
    for item in dataset:
        x_unit_id = item['X_unit_id']
        if x_unit_id != now_id:
            # merge now_items
            now_items = merge_items(now_items)
            merged_dataset.append(now_items)
            now_items = []
            now_id = x_unit_id
            
    
    def merge_items(now_items: list) -> dict:
        res = now_items[0]
        scores ={
            "fluency": np.mean([item['fluency'] for item in now_items]),
            "grammaticality": np.mean([item['grammaticality'] for item in now_items]),
            "semantic_adequacy": np.mean([item['semantic_adequacy'] for item in now_items]),
        }
        res.update(scores)
        return res
    
    dataset = merged_dataset
    
    formated_data = {}
    src_id = 0
    for item in dataset:
        src = item['mr'].replace("|", ",").replace("<br>", "\n").strip()
        if src not in formated_data:
            formated_data[src] = {
                "id": f"webnlg2017_human_eval_{src_id}",
                "instruction": "Generate a description for the following triples.",
                "input": src,
                "refs": [],
                "data_source": "webnlg2017",
                "task": "data2text",
                "candidates": [],
            }
            src_id += 1
        # add candidate and avoid duplicate candidates
        if any([cand['text'] == item['text'] for cand in formated_data[src]['candidates']]):
            continue
        if item['team'] == 'webnlg':
            formated_data[src]['refs'].append(item['text'])
            continue
        formated_data[src]['candidates'].append({
            "text": item['text'],
            "model": item['team'],
            "decoding_method": "greedy",
            "domain": item['category'],
            "type": item['type'],
            "scores": {
                "fluency": item['fluency'],
                "grammaticality": item['grammaticality'],
                "semantic_adequacy": item['semantic_adequacy'],
            }
        })
    train_data = list(formated_data.values())
    logging.info("Train data statistics:")
    logging.info("# Examples: {}".format(len(train_data)))
    logging.info("# Avg. Unique outputs: {}".format(sum([len(x['candidates']) for x in train_data]) / len(train_data)))
    logging.info("# Unique src: {}".format(len(set([x['input'] for x in train_data]))))
    logging.info("Domain distribution:")
    domain_counter = Counter([x['data_source'].split("_")[-1] for x in train_data])
    for domain in domain_counter:
        logging.info("  {}: {}".format(domain, domain_counter[domain]))
    logging.info("Type distribution:")
    type_counter = Counter([x['type'] for x in train_data])
    for type in type_counter:
        logging.info("  {}: {}".format(type, type_counter[type]))
    return train_data

def str2bool(v):
    if isinstance(v, bool):
        return v
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='../../data')
    parser.add_argument('--max_ex_size', type=int, default=1000)
    parser.add_argument('--overwrite', type=str2bool, default=True)
    parser.add_argument('--data_version', type=int, default=2020,choices=[2017,2020])
    args = parser.parse_args()
    
    dataset_name = f"teven/webnlg_f{args.data_version}_human_eval"

    # Load the training data
    logging.info("Loading data from {}".format(dataset_name))
    dataset = load_dataset(dataset_name, split='train')

    if len(dataset) == 0:
        logging.error("error")
        exit(0)
    else:
        logging.info("Loaded {} examples for {}".format(len(dataset), dataset_name))
    # Save the data
    args.data_dir = Path(args.data_dir)
    args.data_dir.mkdir(parents=True, exist_ok=True)
    train_file = args.data_dir / f"webnlg_{args.data_version}/train_data.json"
    train_file.parent.mkdir(parents=True, exist_ok=True)
    if not train_file.exists() or args.overwrite:
        train_data = format_eval_data(dataset, args.data_version)
        with open(train_file, "w") as f:
            json.dump(train_data, f, indent=4, ensure_ascii=False)
            logging.info("Saved training data to {}".format(train_file))
tigerscore/prepare_data/get_d2t_eval_data.py
TIGER-AI-Lab/TIGERScore
# webnlg2020
from pathlib import Path
import json
import os
import sys

submissions_path = os.path.join(os.path.dirname(os.path.realpath("__file__")), '..', '..', 'challenge-2020','submissions','rdf2text','en')
system_outputs = []
system_files = []
system_names = []
for root, dirs, files in os.walk(submissions_path):
    for system_ in dirs:
        system_files.append(open(os.path.join(root, system_, 'primary.en'), 'r').readlines())
        system_names.append(system_)

human_eval_files = {}
human_eval_path = os.path.join(os.path.dirname(os.path.realpath("__file__")), '..', '..', 'challenge-2020','evaluation','human-evaluation','results','en')
for root, dirs, files in os.walk(human_eval_path):
    for system_ in dirs:
        if system_ in system_names:
            human_eval_files[system_] = json.load(open(os.path.join(root, system_, 'primary.json'), 'r'))

# both readlines
for line_num in range(len(system_files[0])):
    system_output = []
    for i,(model_name,model_output) in enumerate(zip(system_names, system_files)):
        scores_dict = {}
        if f"{line_num + 1}" in human_eval_files[model_name]:
            scores_dict = human_eval_files[model_name][f"{line_num + 1}"]
            if scores_dict:
                new_scores_dict = {k:0 for k in list(scores_dict.values())[0].keys()}
                if 'feedback' in new_scores_dict:
                    del new_scores_dict['feedback']
                for scores_one_dict in scores_dict.values():
                    for k,v in scores_one_dict.items():
                        if k != 'feedback':
                            new_scores_dict[k] += v
                for k,v in new_scores_dict.items():
                    new_scores_dict[k] = v / len(scores_dict)
                scores_dict = new_scores_dict
                
        system_output.append({'model': model_name, 
                              "decoding_method": "greedy",
                              'text': model_output[line_num].strip(),
                              "scores":  scores_dict
        })
    system_outputs.append(system_output)
print(len(system_outputs[0]))
print(len(system_outputs))

reference_path = os.path.join(os.path.dirname(os.path.realpath("__file__")), '..', '..', 'challenge-2020','evaluation','references', 'references-en.json')
references = json.load(open(reference_path, 'r'))['entries']
output_data = []
for i,item in enumerate(references):
    item = item[f"{i + 1}"]
    output_data.append({'id': i + 1, 
                        'instruction': "Generate a description for the following triples.",
                        'input': "\n".join([f"({triple['subject']},{triple['property']},{triple['object']})" for triple in item["modifiedtripleset"]]).strip(),
                        'output': list(set([j["lex"] for j in item["lexicalisations"]])),
                        'candidates': system_outputs[i]
                        })
json.dump(output_data, open('webnlg2020_gen.json', 'w'), indent=4,ensure_ascii=False)

output_data_with_scores = []
for i,item in enumerate(references):
    item = item[f"{i + 1}"]
    if system_outputs[i][0]["scores"]:
        output_data_with_scores.append({'id': i + 1, 
                        'instruction': "Generate a description for the following triples.",
                        'input': "\n".join([f"({triple['subject']},{triple['property']},{triple['object']})" for triple in item["modifiedtripleset"]]).strip(),
                        'output': list(set([j["lex"] for j in item["lexicalisations"]])),
                        'candidates': system_outputs[i]
                        })
json.dump(output_data_with_scores, open('webnlg2020_gen_with_scores.json', 'w'), indent=4,ensure_ascii=False)
tigerscore/prepare_data/get_d2t_train_data.py
TIGER-AI-Lab/TIGERScore
from pathlib import Path
import json
import os
import sys

datas = []
tmp_data = {}
for split in ["train", "test"]:
    with open(f'/home//WorkSpace/ExplainableGPTScore_bak/gsm8k-ScRel/data/{split}_use.jsonl', 'r') as f:
        data = f.readlines()
        for line in data:
            line = json.loads(line)
            tmp_data[line["query"]] = {
                "instruction": "Below is an instruction that describes a task. Write a response that appropriately completes the request. Let's think step by step.",
                # "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response: Let's think step by step."
                "input": line["query"],
                "output": line["response"],
                "candidates": []
            }

jsonl_dir = '/home//WorkSpace/ExplainableGPTScore_bak/gsm8k-ScRel/data/rft/'

for root, dirs, files in os.walk(jsonl_dir):
    for file in files:
        with open(jsonl_dir+file, 'r') as f:
            data = f.readlines()
            for line in data:
                line = json.loads(line)
                if line["query"] in tmp_data:
                    tmp_data[line["query"]]["candidates"].append({
                        "model": f"{file.split('.')[0]}",
                        "decoding_method": "greedy",
                        "text": line["response"],
                        "scores": {}
                    })
for k, v in tmp_data.items():
    if len(v["candidates"]) != 0:
        datas.append(v)
# datas = sorted(datas, key=lambda x: x["id"])

output_file = Path('/home//WorkSpace/ExplainableGPTScore_bak/gsm8k-ScRel/data/data_gen.json')
with open(output_file, 'w') as f:
    json.dump(datas, f, indent=4,ensure_ascii=False)
print(len(datas))
print(datas[0])
tigerscore/prepare_data/get_gsm8k_data.py
TIGER-AI-Lab/TIGERScore
"""
Get SUM train data
"""
import json
import argparse
import time
import logging
import numpy as np
from typing import List
from pathlib import Path
from datasets import load_dataset
from pathlib import Path
from collections import Counter
from itertools import chain
from transformers import AutoTokenizer

logging.basicConfig(level=logging.INFO)
# log current time
logging.info("\nRunning time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))

"""
Datasets:
50k data total
    - CNN/DM cnn_dailymail 30000 Yale-LILY/brio-cnndm-uncased
    - XSUM 2000 Yale-LILY/brio-xsum-cased
    - Newsroom 600 BARTScore
    - Samsum 2000
    - Gigaword 2000
    - REALSumm manu/REALSumm 2400 BARTScore
"""
# cnn/dm
from pathlib import Path
import json
import os
import sys

datas = []
tmp_data = {}
with open('/home//WorkSpace/ExplainableGPTScore_bak/data/sum/summeval/M0/paired/outputs.aligned.paired.jsonl', 'r') as f:
    data = f.readlines()
    for line in data:
        line = json.loads(line)
        tmp_data[line["text"]] = {
            "id": line["id"],
            "instruction": "Write a summary of the text below.",
            "input": line["text"],
            "output": line["reference"],
            "candidates": []
        }
        # datas.append({
        #     "id": line["id"],
        #     "instruction": "Write a summary of the text below.",
        #     "input": line["text"],
        #     "output": line["reference"],
        #     "candidates": []
        # })
file_caterogy =[
    "outputs.aligned.paired.jsonl",
    "outputs.aligned.paired.jsonl",
    "outputs.aligned.paired.jsonl",
    "outputs.aligned.paired.jsonl",
    "outputs.aligned.paired.jsonl",
    "outputs_rouge.aligned.paired.jsonl",
    "outputs_cnndm.aligned.paired.jsonl",
    "outputs.aligned.paired.jsonl",
    "outputs_ptrgen.aligned.paired.jsonl",
    "outputs_extabs+rl+rerank.aligned.paired.jsonl",
    "outputs_transformer.aligned.paired.jsonl",
    "outputs_novelty+lm.aligned.paired.jsonl",
    "outputs.aligned.paired.jsonl",
    "outputs.aligned.paired.jsonl",
    "outputs.aligned.paired.jsonl",
    "outputs_coverage.aligned.paired.jsonl",
    "outputs_ext.aligned.paired.jsonl",
    "outputs_large.aligned.paired.jsonl",
    "outputs_abs_bert.aligned.paired.jsonl",
    "outputs_cnndm_bertsumextabs.aligned.paired.jsonl",
    "outputs_rl+supervised.aligned.paired.jsonl",
    "outputs.aligned.paired.jsonl",
    "outputs_cnndm.aligned.paired.jsonl",
    "outputs_dynamicmix_cnn_dailymail.aligned.paired.jsonl",
]
for i, file in enumerate(file_caterogy):
    file_i = '/home//WorkSpace/ExplainableGPTScore_bak/data/sum/summeval/M{}/paired/{}'.format(i, file)
    
    with open(file_i, 'r') as f:
        data = f.readlines()
        for line in data:
            line = json.loads(line)
            try:
                if line["text"] in tmp_data:
                    tmp_data[line["text"]]["candidates"].append({
                        "model": f"M{i}",
                        "decoding_method": "greedy",
                        "text": line["decoded"].replace("-lrb-", "(").replace("-rrb-", ")"), # for cnndm
                        "scores": {}
                    })
            except KeyError:
                continue
for k, v in tmp_data.items():
    datas.append(v)
datas = sorted(datas, key=lambda x: x["id"])

output_file = Path('/home//WorkSpace/ExplainableGPTScore_bak/data/sum/summeval/summ_gen.json')
with open(output_file, 'w') as f:
    json.dump(datas, f, indent=4,ensure_ascii=False)

        

def str2bool(v):
    if isinstance(v, bool):
        return v
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='../../data/sum')
    parser.add_argument('--max_ex_size', type=int, default=1000)
    parser.add_argument('--overwrite', type=str2bool, default=False)
    args = parser.parse_args()
    
    dataset_list = ["cnn_dailymail:3.0.0","xsum","gigaword","samsum"]
tigerscore/prepare_data/get_sum_train_data.py
TIGER-AI-Lab/TIGERScore
import json
import sys
import fire
import random
import logging
from pathlib import Path
from typing import List, Union
from transformers import AutoTokenizer
from tqdm import tqdm
sys.path.append("..")
from common.datasets_config import DATASETS_CONFIG
logging.basicConfig(level=logging.INFO)

def main(
    task: str = "long-form QA",
    data_dir: str = "../../data/",
    shuffle: bool = True,
    seed: int = 42,
    max_size_per_ds: int = 1000,
    max_input_length: int = None,
    max_output_length: int = None,
):
    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
    task_datasets_config = DATASETS_CONFIG[task]
    final_ds = []
    for dataset_name, dataset_config in task_datasets_config.items():
        ds_train_file = Path(data_dir) / dataset_name.replace(":", "/") / "train_data.json"
        with open(ds_train_file, "r") as f:
            ds_train = json.load(f)
        if shuffle:
            random.seed(seed)
            random.shuffle(ds_train)
        if isinstance(max_input_length, int) and max_input_length > 0:
            inputs = [item['instruction'] + "\n" + item["input"] for item in ds_train]
            inputs_ids = [tokenizer.encode(x, add_special_tokens=False) 
                for x in tqdm(inputs, desc=f"Tokenizing {dataset_name} inputs")]
            _ds_train = [x for x, _ids in zip(ds_train, inputs_ids) if len(_ids) <= max_input_length]
            logging.info("Removed {} examples with inst+input length > {}".format(
                len(ds_train) - len(_ds_train), max_input_length))
            ds_train = _ds_train
        if isinstance(max_output_length, int) and max_output_length > 0:
            outputs = [item['output'] for item in ds_train]
            outputs_ids = [tokenizer.encode(x, add_special_tokens=False) 
                for x in tqdm(outputs, desc=f"Tokenizing {dataset_name} outputs")]
            _ds_train = [x for x, _ids in zip(ds_train, outputs_ids) if len(_ids) <= max_output_length]
            logging.info("Removed {} examples with output length > {}".format(
                len(ds_train) - len(_ds_train), max_output_length))
        if isinstance(max_size_per_ds, int) and max_size_per_ds > 0:
            ds_train = ds_train[:max_size_per_ds]
        for item in ds_train:
            item["data_source"] = dataset_name
            item["task"] = task
        final_ds.extend(ds_train)
    logging.info("Final dataset size: {}".format(len(final_ds)))
    task_ds_train_file = Path(data_dir) / "synthesis_debug" / task / "train_data.json"
    task_ds_train_file.parent.mkdir(parents=True, exist_ok=True)
    with open(task_ds_train_file, "w") as f:
        json.dump(final_ds, f, indent=4, ensure_ascii=False)
        logging.info("Saved to {}".format(task_ds_train_file))
    
    
if __name__ == "__main__":
    fire.Fire(main)
    
tigerscore/prepare_data/get_synthesis_data.py
TIGER-AI-Lab/TIGERScore
"""
Get WMT train data
"""
import json
import argparse
import logging
import time
import numpy as np
from typing import List
from pathlib import Path
from datasets import load_dataset
from pathlib import Path
from collections import Counter
from itertools import chain
from transformers import AutoTokenizer
from mt_metrics_eval.data import EvalSet


logging.basicConfig(level=logging.INFO)
# log current time
logging.info("Running time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))


lang_map = {
    'zh': 'Chinese',
    'en': 'English',
    'de': 'German',
    'ru': 'Russian',
    'cs': 'Czech',
    'uk': 'Ukrainian',
    'hr': 'Croatian',
    'ja': 'Japanese',
    'liv': 'Livonian',
}

def get_eval_data(wmt_version, lang_pair, human_score_name):
    src_lang, tgt_lang = lang_pair.split("-")
    logging.info("Loading data from mt_metrics_eval EvalSet for \
        wmt_version={}, lang_pair={}, human_score_name={}".format(wmt_version, lang_pair, human_score_name))
    try:
        data = EvalSet(wmt_version, lang_pair, read_stored_metric_scores=True)
    except FileNotFoundError:
        logging.error("No stored metric scores found for wmt_version={}, lang_pair={}".format(wmt_version, lang_pair))
        return None
    human_scores_names = set(x for x in data.human_score_names if x.startswith(human_score_name))
    load_metrics = ['chrF', 'COMET', "BLEU", "BERTScore", "BLEURT"]
    metrics_names = set(x for x in data.metric_names if any(x.startswith(y) for y in load_metrics))
    valid_sys_names = data.sys_names - data.human_sys_names
    ref_names = data.ref_names
    print("Valid systems: {}".format(valid_sys_names))
    for sys_name in data.sys_names - data.human_sys_names:
        a = [len([x for x in data._scores['seg'][k][sys_name] if x is not None]) for k in human_scores_names.union(metrics_names) if data._scores['seg'][k]]
        for i, metric in enumerate(human_scores_names.union(metrics_names)):
            if a[i] == 0:
                print("Metric {} is missing for sys {}".format(metric, sys_name))
                try:
                    valid_sys_names.remove(sys_name)
                except KeyError:
                    logging.info("Sys {} is already removed".format(sys_name))

    formated_data = []
    for i in range(len(data.src)):
        if any([data._scores['seg'][human_score_name][sys_name][i] is None for sys_name in valid_sys_names]):
            # skip examples without human scores
            continue
        domain = None
        for d in data.domains:
            for start, end in data.domains[d]:
                if start <= i < end:
                    domain = d
                    break
        assert domain is not None
        formated_data.append({
            "id": f"{wmt_version}_{lang_pair}_eval_{i}",
            "instruction": f"Translate the following text from {lang_map[src_lang]} to {lang_map[tgt_lang]}.",
            "input": data.src[i],
            "refs": [data.sys_outputs[ref_name][i] for ref_name in ref_names],
            "data_source": f"{wmt_version}_{lang_pair}_{domain}",
            "task": "translation",
            "candidates": [
                {
                    "text": data.sys_outputs[sys_name][i],
                    "model": sys_name,
                    "decoding_method": "greedy",
                    "scores": {
                        k: data._scores['seg'][k][sys_name][i]
                        for k in human_scores_names.union(metrics_names) if data._scores['seg'][k]
                    },
                }
                for sys_name in valid_sys_names
            ]
        })
    logging.info(f"Eval {lang_pair} data statistics:")
    logging.info("# Examples: {}".format(len(formated_data)))
    logging.info("# Avg. Unique outputs: {}".format(sum([len(x['candidates']) for x in formated_data]) / len(formated_data)))
    logging.info("# Unique src: {}".format(len(set([x['input'] for x in formated_data]))))
    logging.info("Domain distribution:")
    domain_counter = Counter([x['data_source'].split("_")[-1] for x in formated_data])
    for domain in domain_counter:
        logging.info("  {}: {}".format(domain, domain_counter[domain]))
    logging.info("Year distribution:")
    year_counter = Counter([x['data_source'].split("_")[0] for x in formated_data])
    for year in year_counter:
        logging.info("  {}: {}".format(year, year_counter[year]))
    return formated_data
    

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--wmt_version', type=str, default='wmt22')
    parser.add_argument('--lang_pair', type=str, default='zh-en')
    parser.add_argument('--human_score_name', type=str, default='mqm', choices=['mqm', 'wmt-z', 'wmt-appraise-z'])
    parser.add_argument('--data_dir', type=str, default='../../data')
    parser.add_argument('--max_ex_size', type=int, default=3500)
    args = parser.parse_args()
    
    lang_pair = args.lang_pair
    
    # Load the eval data
    eval_data = get_eval_data(args.wmt_version, args.lang_pair, args.human_score_name)
    if eval_data is None:
        logging.error("No eval data found for wmt_version={}, lang_pair={}, human_score_name={}".format(args.wmt_version, args.lang_pair, args.human_score_name))
    else:
        logging.info("Loaded {} eval examples".format(len(eval_data)))

    # Save the data
    args.data_dir = Path(args.data_dir)
    args.data_dir.mkdir(parents=True, exist_ok=True)
    eval_file = args.data_dir / f"{args.wmt_version}/{args.lang_pair}/eval_data.json"
    eval_file.parent.mkdir(parents=True, exist_ok=True)

    if eval_data:
        logging.info("Eval data statistics:")
        logging.info("# Examples: {}".format(len(eval_data)))
        logging.info("# Avg. Unique outputs: {}".format(sum([len(x['candidates']) for x in eval_data]) / len(eval_data)))
        logging.info("# Unique src: {}".format(len(set([x['input'] for x in eval_data]))))
        with open(eval_file, "w") as f:
            json.dump(eval_data, f, indent=4, ensure_ascii=False)
            logging.info("Saved eval data to {}".format(eval_file))
        


    

tigerscore/prepare_data/get_wmt_eval_data.py
TIGER-AI-Lab/TIGERScore
"""
Get WMT train data
"""
import json
import argparse
import time
import logging
import numpy as np
import sys
from typing import List
from pathlib import Path
from datasets import load_dataset
from pathlib import Path
from collections import Counter
from itertools import chain
from transformers import AutoTokenizer
from mt_metrics_eval.data import EvalSet
sys.append("../")
from common.evaluation import overall_eval
logging.basicConfig(level=logging.INFO)
# log current time
logging.info("\nRunning time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))

lang_map = {
    'zh': 'Chinese',
    'en': 'English',
    'de': 'German',
    'ru': 'Russian',
    'cs': 'Czech',
    'uk': 'Ukrainian',
    'hr': 'Croatian',
    'ja': 'Japanese',
    'liv': 'Livonian',
    'fi': 'Finnish',
    'fr': 'French',
}

def format_train_data(dataset, lang_pair, human_score_name):
    src_lang, tgt_lang = lang_pair.split("-")
    formated_data = {}
    src_id = 0
    for item in dataset:
        year = item['year'] % 100
        src = item['src']
        if src not in formated_data:
            formated_data[src] = {
                "id": f"wmt{year}_{lang_pair}_train_{src_id}",
                "instruction": f"Translate the following text from {lang_map[src_lang]} to {lang_map[tgt_lang]}.",
                "input": item['src'],
                "refs": [item['ref']],
                "data_source": f"wmt{year}_{lang_pair}_{item['domain']}",
                "task": "translation",
                "candidates": []
            }
            src_id += 1
        # add candidate and avoid duplicate candidates
        if any([cand['text'] == item['mt'] for cand in formated_data[src]['candidates']]):
            continue
        formated_data[src]['candidates'].append({
            "text": item['mt'],
            "model": item['system'] if "system" in item else "unknown",
            "decoding_method": "greedy",
            "domain": item['domain'],
            "scores": {
                human_score_name: item['score']
            }
        })
        if item['ref'] not in formated_data[src]['refs']:
            formated_data[src]['refs'].append(item['ref'])
    train_data = list(formated_data.values())
    logging.info("Train data statistics:")
    logging.info("# Examples: {}".format(len(train_data)))
    logging.info("# Avg. Unique outputs: {}".format(sum([len(x['candidates']) for x in train_data]) / len(train_data)))
    logging.info("# Unique src: {}".format(len(set([x['input'] for x in train_data]))))
    logging.info("Domain distribution:")
    domain_counter = Counter([x['data_source'].split("_")[-1] for x in train_data])
    for domain in domain_counter:
        logging.info("  {}: {}".format(domain, domain_counter[domain]))
    logging.info("Year distribution:")
    year_counter = Counter([x['data_source'].split("_")[0] for x in train_data])
    for year in year_counter:
        logging.info("  {}: {}".format(year, year_counter[year]))
    return train_data

def down_sample_train_data(train_data: List[dict], max_ex_size: int):
    """
    Down sample the formatted training data to have at most max_uni_outputs unique outputs
    Args:
        train_data: formatted training data
        max_ex_size: max total number of candidate outputs
    """
    # randomly downsample
    total_num_cands = sum([len(x['candidates']) for x in train_data])
    np.random.seed(42)
    cands_idxs = np.arange(total_num_cands)
    np.random.shuffle(cands_idxs)
    sampled_cands_idxs = cands_idxs[:max_ex_size]
    sampled_cands_idxs.sort()
    sampled_train_data = []
    cur_cand_idx = 0
    for i, ex in enumerate(train_data):
        sampled_ex = {
            "id": ex['id'],
            "instruction": ex['instruction'],
            "input": ex['input'],
            "refs": ex['refs'],
            "data_source": ex['data_source'],
            "task": ex['task'],
            "candidates": []
        }
        for j, cand in enumerate(ex['candidates']):
            if cur_cand_idx in sampled_cands_idxs:
                sampled_ex['candidates'].append(cand)
            cur_cand_idx += 1
        if len(sampled_ex['candidates']) > 0:
            sampled_train_data.append(sampled_ex)
    # report statistics
    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf") 
    cand_token_ids = [[tokenizer.encode(x['text'], add_special_tokens=False) for x in ex['candidates']] for ex in sampled_train_data]
    cand_token_lens = [[len(x) for x in cand_token_id] for cand_token_id in cand_token_ids]
    cand_token_lens = list(chain(*cand_token_lens))
    
    logging.info("Downsampled train data statistics:")
    logging.info("# Examples: {}".format(len(sampled_train_data)))
    logging.info("# Avg. Unique outputs: {}".format(sum([len(x['candidates']) for x in sampled_train_data]) / len(sampled_train_data)))
    logging.info("# Avg. Candidates Min/Mean/Max Length: {}".format(
        np.min(cand_token_lens), np.mean(cand_token_lens), np.max(cand_token_lens)))
    logging.info("Domain distribution:")
    domain_counter = Counter([x['data_source'].split("_")[-1] for x in sampled_train_data])
    for domain in domain_counter:
        logging.info("  {}: {}".format(domain, domain_counter[domain]))
    logging.info("Year distribution:")
    year_counter = Counter([x['data_source'].split("_")[0] for x in sampled_train_data])
    for year in year_counter:
        logging.info("  {}: {}".format(year, year_counter[year]))

    return sampled_train_data

def str2bool(v):
    if isinstance(v, bool):
        return v
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--lang_pair', type=str, default='zh-en')
    parser.add_argument('--human_score_name', type=str, default='mqm', choices=['mqm', 'da'])
    parser.add_argument('--data_dir', type=str, default='../../data')
    parser.add_argument('--max_ex_size', type=int, default=1000)
    parser.add_argument('--overwrite', type=str2bool, default=True)
    args = parser.parse_args()
    
    dataset_name = f"RicardoRei/wmt-{args.human_score_name}-human-evaluation"
    lang_pair = args.lang_pair

    # Load the training data
    logging.info("Loading data from {}".format(dataset_name))
    dataset = load_dataset(dataset_name, split='train')
    dataset = dataset.filter(lambda x: x['lp'] == lang_pair) # filter by lang_pair
    dataset = dataset.filter(lambda x: x['year'] < 2022) # only use data before WMT22

    if len(dataset) == 0:
        logging.error("No data found for lang_pair={}".format(lang_pair))
        exit(0)
    else:
        logging.info("Loaded {} examples for lang_pair={}".format(len(dataset), lang_pair))
    # Save the data
    args.data_dir = Path(args.data_dir)
    args.data_dir.mkdir(parents=True, exist_ok=True)
    train_file = args.data_dir / f"wmt/{args.lang_pair}/train_data.json"
    train_file.parent.mkdir(parents=True, exist_ok=True)
    if not train_file.exists() or args.overwrite:
        train_data = format_train_data(dataset, lang_pair, args.human_score_name)
        train_data = down_sample_train_data(train_data, args.max_ex_size)
        with open(train_file, "w") as f:
            json.dump(train_data, f, indent=4, ensure_ascii=False)
            logging.info("Saved training data to {}".format(train_file))
tigerscore/prepare_data/get_wmt_train_data.py
TIGER-AI-Lab/TIGERScore
import os,sys,json
from pathlib import Path
import jsonlines 

input_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/llm-blender/mix-instruct/test_data_prepared.jsonl"
output_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/llm-blender/mix-instruct/test_data_prepared.json"

with jsonlines.open(input_file) as reader:
    data = [obj for obj in reader]
    
new_data = []
for d in data:
    new_data.append({
        "id": d["id"],
        "instruction": d["instruction"] if d["instruction"] else "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.",
        "input": d["input"],
        "output": d["output"],
        "candidates" : d["candidates"],
        "cmp_results": d["cmp_results"],
    })

with open(output_file, 'w') as outfile:
    json.dump(new_data, outfile)

print(len(data))

import numpy as np

def get_ranks_from_chatgpt_cmps(ds_data):

    # transform chatgpt cmp_results to [bz, c, c]
    bz = len(ds_data)
    c = len(ds_data[0]['candidates'])

    chatgpt_cmp_results = np.zeros((bz, c, c))
    _models = [c['model'] for c in ds_data[0]['candidates']]
    for i, d in enumerate(ds_data):
        models = [c['model'] for c in d['candidates']]
        assert models == _models, f"models not match: {models} vs {_models}"
        if isinstance(d['cmp_results'],str):
            if d['cmp_results'] == "null":
                continue
            else:
                d['cmp_results'] = eval(d['cmp_results'])
        
        if isinstance(d['cmp_results'], dict):
            for cand in d['candidates']:
                cand["scores"]["gpt_rank_score"] = 0
            for key, value in d['cmp_results'].items():
                idx1, idx2 = models.index(key.split(",")[0]), models.index(key.split(",")[1])
                if value == "A is better":
                    for cand in d['candidates']:
                        if cand['model'] == key.split(",")[0]:
                            cand["scores"]["gpt_rank_score"] += 1
                        if cand['model'] == key.split(",")[1]:
                            cand["scores"]["gpt_rank_score"] -= 1
                    chatgpt_cmp_results[i][idx1][idx2] += 1
                    chatgpt_cmp_results[i][idx2][idx1] -= 1
                elif value == "B is better":
                    for cand in d['candidates']:
                        if cand['model'] == key.split(",")[0]:
                            cand["scores"]["gpt_rank_score"] -= 1
                        if cand['model'] == key.split(",")[1]:
                            cand["scores"]["gpt_rank_score"] += 1
                    chatgpt_cmp_results[i][idx1][idx2] -= 1
                    chatgpt_cmp_results[i][idx2][idx1] += 1
                elif value == "Same good":
                    for cand in d['candidates']:
                        if cand['model'] == key.split(",")[0]:
                            cand["scores"]["gpt_rank_score"] += 0.5
                        if cand['model'] == key.split(",")[1]:
                            cand["scores"]["gpt_rank_score"] += 0.5
                    chatgpt_cmp_results[i][idx1][idx2] += 0.5
                    chatgpt_cmp_results[i][idx2][idx1] += 0.5
                elif value == "Same bad":
                    for cand in d['candidates']:
                        if cand['model'] == key.split(",")[0]:
                            cand["scores"]["gpt_rank_score"] -= 0.5
                        if cand['model'] == key.split(",")[1]:
                            cand["scores"]["gpt_rank_score"] -= 0.5
                    chatgpt_cmp_results[i][idx1][idx2] -= 0.5
                    chatgpt_cmp_results[i][idx2][idx1] -= 0.5
                else:
                    raise ValueError("Unknown value: {}".format(value))
                
            
        else:
            print(d['cmp_results'])

    chatgpt_cmp_ranks = get_ranks_from_cmps(chatgpt_cmp_results)

    model_ranks_map = {}
    for i, model_name in enumerate(_models):
        model_ranks_map[model_name] = chatgpt_cmp_ranks[:, i]
    return model_ranks_map, chatgpt_cmp_results,ds_data


def get_ranks_from_cmps(cmp_results, policy="max_logits"):
    """
    Args:
        cmp_results: ndarray of shape (n, c, c) where n is the number of samples, c is the number of candidates
            for each element, >0 means the first candidate is better than the second one, <0 means the second one is better
    Returns:
        ranks: ndarray of shape (n, c) where n is the number of samples, c is the number of candidates
    """
    if isinstance(cmp_results, list):
        cmp_results = np.array(cmp_results)
    bz, c, _ = cmp_results.shape
    ranks = np.zeros((bz, c), dtype=np.int32)
    for i in range(bz):
        if policy == "max_logits":
            scores = (cmp_results[i] - cmp_results[i].T).sum(axis=-1)
        elif policy == "max_wins":
            scores = (cmp_results[i] > 0).sum(axis=-1) + (cmp_results[i] < 0).sum(axis=-2)
        _ranks = get_ranks_from_scores(scores)
        ranks[i] = _ranks
    return ranks

def get_ranks_from_scores(scores):
    """
    Args:
        scores: ndarray of shape (n, c) or (c) where n is the number of samples, c is the number of candidates
        Treat same as higher one
        
    Returns:
        ranks: ndarray of shape (n, c) or (c) where n is the number of samples, c is the number of candidates
    """
    if isinstance(scores, list):
        scores = np.array(scores)
    orig_shape = scores.shape
    if len(scores.shape) == 1:
        scores = scores.reshape(1, -1)
    bz, c = scores.shape
    ranks = np.zeros((bz, c), dtype=np.int32)
    for i in range(bz):
        sorted_scores_i = list(sorted(list(scores[i]), reverse=True))
        for j in range(c):
            ranks[i, j] = sorted_scores_i.index(scores[i, j]) + 1
    
    ranks = ranks.reshape(orig_shape)
    return ranks


_,_,new_data=get_ranks_from_chatgpt_cmps(data)

output_new_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/llm-blender/mix-instruct/test_data_new_prepared.json"
with open(output_new_file, 'w') as outfile:
    json.dump(new_data, outfile, indent=4, ensure_ascii=False)
    
output_new_file = "/home//WorkSpace/ExplainableGPTScore_bak/data/llm-blender/mix-instruct/test_data_new_prepared.json"
with open(output_new_file, 'r') as outfile:
    data = json.load(outfile)
    
need_remove = []
    
for item in data:
    for cand in item['candidates']:
        if "gpt_rank_score" not in cand["scores"]:
            need_remove.append(item)
            break

for item in need_remove:
    data.remove(item)
    
with open(output_new_file, 'w') as outfile:
    json.dump(data, outfile, indent=4, ensure_ascii=False)
tigerscore/prepare_data/load_mix-instruct.py
TIGER-AI-Lab/TIGERScore
import os,sys,json
import numpy as np
import shutil
mix_data = {
    "summarization": "/home//WorkSpace/ExplainableGPTScore_bak/data/sum/train_data_new.align_score.filter_v2.json",
    "data2text": "/home//WorkSpace/ExplainableGPTScore_bak/data/d2t/train_data_new.d2t.distill_new.filter_v3.json",
    "translation" : "/home//WorkSpace/ExplainableGPTScore_bak/data/wmt/train_data.wmt.distill.filter_v3.json",
    "long-form QA": "/home//WorkSpace/ExplainableGPTScore_bak/data/lfqa/train_data_new.longform_qa.v3.json",
    "instruction-following": "/home//WorkSpace/ExplainableGPTScore_bak/data/ins/train_data_new.instruction_following_new.v3.json"
}


for task in mix_data:
    # copy data to mix_data.json
    shutil.copy(mix_data[task], f"/home//WorkSpace/ExplainableGPTScore_bak/data/real_world/{task}.json")
        
        
# print(len(data))
tigerscore/prepare_data/mix_data.py
TIGER-AI-Lab/TIGERScore
# %%
import json,os,sys
import numpy as np

roc_input_file_path = "/home//WorkSpace/ExplainableGPTScore_bak/OpenMEVA/benchmark/data/mans_roc.json"
with open(roc_input_file_path, 'r') as f:
    data = json.load(f)

new_data = []
for key, value in data.items():
    cands = []
    for _k, _v in value["gen"].items():
        cands.append({
            "model": _k,
            "decoding_method": "greedy",
            "text": _v["text"],
            "score": {"human":np.mean(_v["score"])},
        })
    new_data.append({
        "id": key,
        "instruction": "Generate a reasonable ending for the following story.",
        "input": value["prompt"],
        "output": value["gold_response"],
        "candidates": cands,
        "data_source": "roc",
    })
    
wp_input_file_path = "/home//WorkSpace/ExplainableGPTScore_bak/OpenMEVA/benchmark/data/mans_wp.json"
with open(wp_input_file_path, 'r') as f:
    data = json.load(f)
for key, value in data.items():
    cands = []
    for _k, _v in value["gen"].items():
        cands.append({
            "model": _k,
            "decoding_method": "greedy",
            "text": _v["text"],
            "score": {"human":np.mean(_v["score"])},
        })
    new_data.append({
        "id": key,
        "instruction": "Generate a reasonable ending for the following story.",
        "input": value["prompt"],
        "output": value["gold_response"],
        "candidates": cands,
        "data_source": "wp",
    })
    
output_file_path = "/home//WorkSpace/ExplainableGPTScore_bak/data/storygen/test.json"
with open(output_file_path, 'w') as f:
    json.dump(new_data, f, indent=4,ensure_ascii=False)
    



tigerscore/prepare_data/openmeva.py
TIGER-AI-Lab/TIGERScore
import torch
import regex as re
from transformers import AutoTokenizer, AutoModelForCausalLM
from string import Template
from typing import List
from tqdm import tqdm


TIGERScore_model_map = {
    "7b": "TIGER-Lab/TIGERScore-7B-V1.0",
    "13b": "TIGER-Lab/TIGERScore-13B-V1.0",
}

FINETUNE_INST = "You are evaluating errors in a model-generated output for a(an) ${task} task."
FINETUNE_INPUT = """\
Task instruction: ${generation_instruction}
Source: ${input_context}
Model-generated Output: ${hypothesis_output}

Based on the given task instruction and source, identify errors in this model-generated output.
For each error you give in the response, please also elaborate the following information:
- error location (the words that are wrong in the output)
- error aspect it belongs to.
- explanation why it's an error, and the correction suggestions.
- severity of the error ("Major" or "Minor"). 
- reduction of score (between 0.5 and 5 given the severity of the error)

Your evaluation output:
"""

class TIGERScorer(object):  
    def __init__(self, model_size, quantized=False):
        """Initialize the TIGERScore model.

        Args:
            model_size:
                either "7b" or "13b"
            quantized:
                If true, load the 4-bit quantized version of the model.
                quantized version occupies 2-3 times less memory but will running slower.
        
        """
        assert model_size in TIGERScore_model_map
        model_name = TIGERScore_model_map[model_size]
        if quantized:
            self.model = AutoModelForCausalLM.from_pretrained(
                model_name,
                torch_dtype=torch.bfloat16,
                device_map="auto",
                load_in_4bit=True,
            )
        else:
            self.model = AutoModelForCausalLM.from_pretrained(
                model_name,
                torch_dtype=torch.bfloat16,
                device_map="auto"
            )
        self.tokenizer = AutoTokenizer.from_pretrained(
            model_name,
            use_fast=True,
            padding_side="left",
        )
    
    def decode_tigerscore_output(self, output):
        """Decode the output of TIGERScore model into structured error explanations.

        Args:
            output (str):
                the output of TIGERScore model.
        Returns:
            errors (List[Dict]):
                structured error explanations for each error in the output.
                Each error explanation is a dictionary with the following fields:
                    - error_location (str): the words that are wrong in the output
                    - error_aspect (str): the aspect of the error
                    - error_explanation (str): explanation why it's an error, and the correction suggestions
                    - error_severity (str): severity of the error ("Major" or "Minor")
                    - score_reduction (float): reduction of score (between 0.5 and 5 given the severity of the error)
                There can be multiple errors in each input.
        """
        result = {}
        result['num_errors'] = re.search(r"(?<=The model-generated output contains )\d+(?= errors)", output).group(0)
        result['score'] = re.search(r"(?<=, with a total score reduction of )\d+", output).group(0)
        result['num_errors'] = int(result['num_errors'])
        result['score'] = -float(result['score'])
        result['errors'] = {}
        error_locations = re.findall(r"(?<=Error location \d+: ).*?(?=\n)", output)
        error_aspects = re.findall(r"(?<=Error aspect \d+: ).*?(?=\n)", output)
        error_explanations = re.findall(r"(?<=Explanation \d+: ).*?(?=\n)", output)
        error_severities = re.findall(r"(?<=Severity \d+: ).*?(?=\n)", output)
        score_reductions = re.findall(r"(?<=\nScore reduction \d+: )(\d+\.\d+|\d+)", output)
        assert len(error_locations) == len(error_aspects) == len(error_explanations) == len(error_severities) == len(score_reductions), \
            "The number of errors does not match."
        for i in range(len(error_locations)):
            error = {}
            error['location'] = error_locations[i]
            error['aspect'] = error_aspects[i]
            error['explanation'] = error_explanations[i]
            error['severity'] = error_severities[i]
            error['score_reduction'] = score_reductions[i]
            result['errors'][f"error_{i}"] = error
        return result
    
    def _score_batch(
        self,
        tasks: List[str],
        insts: List[str],
        input_contexts: List[str],
        hypo_outputs: List[str],
        **generate_kwargs
    ):
        """Internal function to score a batch of inputs.
        Args:
            (See score() function)
        Returns:
            (See score() function)
        """
        assert len(tasks) == len(insts) == len(input_contexts) == len(hypo_outputs)
        inst_template = Template(FINETUNE_INST)
        input_template = Template(FINETUNE_INPUT)

        insts = [inst_template.substitute(task=task) for task in tasks]
        inputs = [
            input_template.substitute(
                generation_instruction=inst,
                input_context=input_context,
                hypothesis_output=hypo_output
            )
            for inst, input_context, hypo_output in zip(insts, input_contexts, hypo_outputs)
        ]
        prompts = [(inst + "\n" + input_part).strip("\n ") + "\n" for inst, input_part in zip(insts, inputs)]
        
        encodings = self.tokenizer(prompts, return_tensors="pt", padding=True, truncation=True, max_length=self.tokenizer.model_max_length)
        input_ids = encodings["input_ids"].to(self.model.device)
        attention_mask = encodings["attention_mask"].to(self.model.device)
        gen_params = {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            "max_new_tokens": 512,
            "do_sample": True,
            "top_k": 1,
            "num_return_sequences": 1,
        }
        gen_params.update(generate_kwargs)
        outputs = self.model.generate(**gen_params)
        
        input_len = input_ids.shape[1]
        # completion_ids = [output[input_len:] for output in outputs]
        completion_ids = outputs
        completions = [self.tokenizer.decode(completion, skip_special_tokens=True) for completion in completion_ids]
        tigerscore_results = []
        for completion in completions:
            try:
                result = self.decode_tigerscore_output(completion)
                result['score'] = result['score']
                result['num_errors'] = result['num_errors']
                result['errors'] = result['errors']
            except:
                result = {}
                result['score'] = None
                result['num_errors'] = None
                result['errors'] = None
            result['raw_output'] = completion
            tigerscore_results.append(result)
        return tigerscore_results

    
    def score(
        self,
        tasks: List[str],
        insts: List[str],
        input_contexts: List[str],
        hypo_outputs: List[str],
        batch_size: int = 8,
        **generate_kwargs
    ):
        """Score and identify errors in the model-generated outputs

        Example Usage:
        ```python
            import os
            os.environ["CUDA_VISIBLE_DEVICES"] = "0"
            from datasets import load_dataset
            from tigerscore import TIGERScorer
            scorer = TIGERScorer(model_size="7b", quantized=True)
            dataset = load_dataset("TIGER-Lab/MetricInstruct")
            num_few_examples = 10
            tasks = dataset["train_mix"]['task'][0:num_few_examples]
            insts = dataset["train_mix"]['instruction'][0:num_few_examples]
            input_contexts = dataset["train_mix"]['input_context'][0:num_few_examples]
            hypo_output = dataset["train_mix"]['hypo_output'][0:num_few_examples]
            results = scorer.score(tasks, insts, input_contexts, hypo_output)
            scores = [result["score"] for result in results]
            print(results)
        ```

        Args:
            tasks:
                a list of tasks, each task is a string. 
                6 pre-defined tasks are:
                    - "translation"
                    - "summarization"
                    - "data2text"
                    - "mathQA"
                    - "long-form QA"
                    - "instruction-following"
                You can also define your own task.
            insts:
                a list of instruction strings; One instruction example is:
                "Translate the following text from German to English."
                A instruction is a short description of the task. 
                It contains specific requirements for the model-generated output.
            input_contexts:
                a list of input contexts; One input context example is the source German text.
            hypo_outputs:
                a list of hypothesis outputs; One hypothesis output example is the model-generated English translation.
            batch_size:
                batch size for scoring. 
            generate_kwargs:
                keyword arguments for the model.generate() method. 
                See https://huggingface.co/transformers/main_classes/model.html
        Returns:
            results (List[Dict]):
                Contains the following fields:
                - score (float): the TIGERScore score for the input.
                - num_errors (int): the number of errors in the input.
                - errors (List[Dict]): structured error explanations for each error in the input.
                    - location (str): the words that are wrong in the output
                    - aspect (str): the aspect of the error
                    - explanation (str): explanation why it's an error, and the correction suggestions
                    - severity (str): severity of the error ("Major" or "Minor")
                    - reduction (float): reduction of score (between 0.5 and 5 given the severity of the error)
                - raw_output (str): the raw output of the TIGERScore model.
        """
        assert len(tasks) == len(insts) == len(input_contexts) == len(hypo_outputs)
        results = []
        for i in tqdm(
            range(0, len(tasks), batch_size), 
            desc="TIGERScore Batch Scoring",
            total=len(tasks)//batch_size + 1
            ):
            batch_tasks = tasks[i:i+batch_size]
            batch_insts = insts[i:i+batch_size]
            batch_input_contexts = input_contexts[i:i+batch_size]
            batch_hypo_outputs = hypo_outputs[i:i+batch_size]
            batch_results = self._score_batch(
                batch_tasks, batch_insts, batch_input_contexts, batch_hypo_outputs, **generate_kwargs
            )
            results.extend(batch_results)
        return results
tigerscore/scorer/tigerscore.py
TIGER-AI-Lab/TIGERScore
EVAL_ASPECTS = {}

# EVAL_ASPECTS["summarization"] = {
#     "Relevance": "How well the summary captures the **key** points of the input text. Consider whether all and **only** the important aspects are contained in the summary, not all the details.",
#     "Redundancy": "Whether the summary contains redundant information. Consider whether the summary contains information that is not necessary to understand the input text.",
#     "Consistency": "Whether the facts in the summary are consistent with the facts in the original article. Consider whether the summary does reproduce all facts accurately and does not make up untrue information.",
#     "Informativeness": "How well the summary is informative. Consider whether the summary is helpful for humans to understand the input text.",
#     "Fluency": "How well the summary is written in terms of grammar, spelling, punctuation, and coherence.",
#     # "Novelty": "The degree to which the summary presents new information or insights not explicitly stated in the original text. This aspect evaluates how well the summary provides new and valuable information not already conveyed in the source."
# }

# EVAL_ASPECTS["summarization"] = {
# "Relevance": "How well the summary captures the key points of the input text.",
# "Consistency": "Whether the facts in the summary are consistent with the facts in the input text.",
# # "Consistency": "Whether the facts in the summary are consistent with the facts in the input text.",
# # "Informativeness": "How well the summary is informative. Consider whether the summary is helpful for humans to understand the input text.",
# # "Coherence": "The quality of all sentences collectively, to the fit together and sound naturally.",
# "Fluency": "The quality of individual sentences, are they well-written and grammatically correct.",
# # "Redundancy": "Whether the summary contains redundant information. Consider whether the summary contains information that is not necessary to understand the input text.",
# }

# EVAL_ASPECTS["summarization"] = {
# "Relevance": "How well the summary captures the key points of the input text.",
# # "Fact Consistency": "Whether the facts in the summary are consistent with the facts in the input text.",
# "Consistency": "The factual alignment between the summary and the summarized source.",
# # "Informativeness": "How well the summary is informative. Consider whether the summary is helpful for humans to understand the input text.",
# "Coherence": "The collective quality of all sentences.",
# "Fluency": "The quality of individual sentences.",
# # "Redundancy": "Whether the summary contains redundant information. Consider whether the summary contains information that is not necessary to understand the input text.",
# }

EVAL_ASPECTS["summarization"] = {
"Relevance": "This aspect refers to the degree to which the summarized output accurately reflects the key points of the input text. It assesses if the summary contains the necessary information from the original text. Example error types include: (1) Including irrelevant information, (2) Omitting important information, (3) Overemphasis on less important details, (4) Under-representation of key points, (5) Misrepresentation of main ideas, etc.",
"Fact Consistency": "This aspect evaluates if the facts in the summary are consistent with the facts in the original text. It checks the accuracy of information presented in the summary. Example error types include: (1) Factual inaccuracies, (2) Misinterpretation, (3) Addition of non-existent facts, (4) Omission of factual information, etc.",
"Coherence": "This aspect pertains to the logical and meaningful arrangement of ideas in the summary. It checks if the summary makes sense as a standalone text. Example error types include: (1) Illogical sequence of ideas, (2) Lack of clear connections between ideas, (3) Poor paraphrasing, (4) Lack of thematic unity, etc.",
"Fluency": "This aspect reviews the model-generated output's use of language, including grammar, punctuation, and vocabulary that affect the quality of the sentences. Example error types include: (1) Spelling mistakes, (2) Grammatical errors, (3) Vocabulary misuse, (4) Style and tone inconsistencies, (5) Garbled characters, etc.",
}

# EVAL_ASPECTS["translation"] = {
# #    # "Missing Translation": "Measure by words that are not translated in the output.",
# #     "Extra Translation": "Measure by words that are translated but not in the input.",
# # "Semantic Shift": "Measured by the extent to which the meaning of a sentence or phrase changes during the translation process despite all words being translated correctly.",
#     "Error Translation": "Measure by words that are translated incorrectly in the output, or words that are not translated in the output, or words that are translated but not in the input.",
#     "Incoherent": "Measured by words that are translated **correctly** but the concatenated output is not a coherent sentence in the target language.",
#     "Style Matching": "Measured by the extent to which the output is consistent with the desired style or genre of the user or the task.",
# }
# EVAL_ASPECTS["translation"] = {
#     "Accuracy": "Measured by the errors where words that additionally, inaccurately, or not translated in the output.",
#     "Fluency": "Measured by the errors in spelling, grammar, punctuation, register, inconsistency, and character encoding, etc. in the output.",
#     "Terminology": "Measured by the errors in terminology, such as the use of proper nouns, technical terms, and abbreviations, etc. And the locale conventions, such as date, time, currency, and number formats, etc.",
#     "Style Matching": "Measured by the extent to which the output is consistent with the desired style or genre of the user or the task.",
# }
# EVAL_ASPECTS["translation"] = {
#     "Accuracy": "Measured by identifying errors in the translation, such as addition of information not present in the source, omission of content from the source, mistranslation that inaccurately represents the source, and untranslated text where portions of the source text are not translated in the output.",
#     "Fluency": "Measured by the errors in translation, including spelling mistakes, grammar issues (excluding orthography), incorrect register, internal inconsistencies (unrelated to terminology), and garbled characters from incorrect encoding",
#     "Terminology": "Measured by the errors in terminology, such as the use of proper nouns, technical terms, and abbreviations, etc. And the locale conventions, such as date, time, currency, and number formats, etc.",
#     "Style Matching": "Measured by the extent to which the output is consistent with the desired style or genre of the user or the task.",
# }
EVAL_ASPECTS["translation"] = {
    "Accuracy": "This aspect refers to the degree to which the translated text adheres to the original text, maintaining the same meaning, context and cultural nuances. Example error types include: (1) Literal translation, (2) Omission, (3) Addition, (4) Mistranslation, (5) False friends , etc.",
    "Fluency": "This aspect refers to how naturally the translation reads in the target language. It should not sound like a translation but like a text originally written in the target language. Example error types include: (1) Grammar mistakes, (2) Syntax errors, (3) Inappropriate language registe, (4) Use of unnatural expressions, (5) Inconsistent tense usage, etc.",
    "Terminology": "This aspect refers to the appropriate use of specific terms and jargon related to a particular field or industry. Example error types include: (1) Incorrect use of proper nouns, (2) Incorrect use of technical terms, (3) Incorrect use of abbreviations, (4) Incorrect date, time, currency, and number formats, etc.",
    "Style Matching": "This aspect refers to the translator's ability to maintain the same style, tone, and voice as the original text. Example error types include: (1) Inconsistent style, (2) Inconsistent tone, (3) Inconsistent voice, etc.",
}

# EVAL_ASPECTS["data2text"] = {
#     # "Fact Completeness": "Measure by the percentage of specified facts that are adequately captured in the generated sentences.",
#     # "Non-Redundancy": "Measure by the percentage of generated sentences that are not redundant.",
#     # May we should give more aspects for data2text
#     #  "Information Completeness": "Measured by the percentage of generated answers that include all the relevant information provided in the input text.",
#     "Relevance": "How well the output captures the key points of the input text.",
#     # "Logical Coherence": "Measure by the percentage of generated sentences that are coherent and logical.",
#     "Fluency": "The quality of sentences.",
# }
EVAL_ASPECTS["data2text"] = {
    "Accuracy": "This aspect deals with the correctness of the information presented by the output. It checks if the output is factually accurate and matches with the information provided in the input. Example error types include: (1) Factual inaccuracies, (2) Misinterpretation, (3) Inaccurate Data Conversion, (4) Contradictory information, (5) Incorrect reasoning, etc.",
    "Logical Coherence": "This aspect evaluates how well the output transforms structured data into a comprehensible, logical, and engaging text. Example error types include: (1) Illogical sequence, (2) Inconsistent facts, (3) Poor paraphrasing, (4) Inadequate explanation, (5) Overcomplication, etc.",
    "Fluency": "This aspect reviews the model-generated output's use of language, including grammar, punctuation, and vocabulary that affect the quality of the sentences. Example error types include: (1) Spelling mistakes, (2) Grammatical errors, (3) Vocabulary misuse, (4) Style and tone inconsistencies, (5) Garbled characters, etc.",
}

EVAL_ASPECTS["mathQA"] = {
    "Problem Understanding": "This aspect assesses how well the output accurately comprehend the text-based description of the math problem. Example error types include: (1) Problem misinterpretation, (2) Neglect of information, (3) Incorrect identification of variables, (4) Overinterpretation (5) Incorrect Assumptions, etc.",
    "Problem Formulation": "This aspect involves translating the problem from a textual form into a mathematical equation or set of equations that can be solved. Example error types include: (1) Incorrect formula usage, (2) Incorrect variable usage, (3) Incorrect operator usage, (4) Missing steps, (5) Misalignment with problem, etc.",
    "Computing Accuracy": "This aspect assesses the output's ability to perform the mathematical operations accurately to arrive at the correct solution. Example error types include: (1) Arithmetic errors, (2) Incorrect variable assignment, (3) Incorrect operation order, (4) Misuse of operation rules, etc.",
    "Solution Interpretation": "This involves the how well the output correctly interpret the solution of the problem in the context of the original problem statement. Example error types include: (1) Incorrect contextual interpretation, (2) Ignoring units of measurement (3) Not addressing all parts of the problem, etc.",
}

# EVAL_ASPECTS["long-form QA"] = {
#     "Fact Accuracy": "Measured by the percentage of generated answers that accurately reflect the facts retrieved from the knowledge source.",
#     # "Fact Missing": "Measured by the percentage of generated answers that are missing some facts retrieved from the knowledge source.",
#     "Specific": "Measured by the percentage of generated answers that are detailed, specific, and step-by-step.",
#     "Coherence": "Measured by the percentage of generated answers that are coherent and logical.",
#     "Informativeness": "Measured by the percentage of generated answers that are helpful in answering the question.",
# }
EVAL_ASPECTS["long-form QA"] = {
    "Accuracy": "This aspect evaluates the factual correctness of the answer. It checks if the information provided in the response is true and matches with the facts available from reliable sources. Example error types include: (1) Factual inaccuracies, (2) Misinterpretation, (3) Outdated information, (4) Contradictory information, (5) Incorrect inference, etc.",
    "Completeness": "This aspect checks if the response provided answers all parts of the question comprehensively. It evaluates if the answer leaves out any critical parts or details that were asked in the question. Example error types include: (1) Partial answer, (2) Omission of key details, (3) Lack of context, (4) Lack of depth, (5) Ignoring multi-part questions, etc.",
    "Informativeness": "This aspect assesses the quality of the response in terms of how helpful it is for the user to understand the answer. It checks if the response is clear, concise, and easy to understand. Example error types include: (1) Redundancy, (2) Irrelevance, (3) Overuse of jargon, (4) Lack of examples, (5) Lack of sources, etc.",
    "Clarity": "This aspect assesses the readability and understandability of the response. Errors in this stage could lead to answers that are difficult to understand, despite being factually correct and based on accurate information synthesis. Example error types include: (1) Unclear language, (2) Poor organization, (3) Lengthy responses, (4) Unexplained jargon, (5) Grammar or spelling errors, etc.",
}
# EVAL_ASPECTS["long-form QA"] = {
#     "Accuracy": "This aspect evaluates the factual correctness of the answer. It checks if the information provided in the response is true and matches with the facts available from reliable sources. Example error types include: (1) Factual inaccuracies, (2) Misinterpretation, (3) Outdated information, (4) Contradictory information, (5) Incorrect inference, etc.",
#     # "Completeness": "This aspect checks if the response provided answers all parts of the question comprehensively. It evaluates if the answer leaves out any critical parts or details that were asked in the question. Example error types include: (1) Partial answer, (2) Omission of key details, (3) Lack of context, (4) Lack of depth, (5) Ignoring multi-part questions, etc.",
#     # "Informativeness": "This aspect assesses the quality of the response in terms of how helpful it is for the user to understand the answer. It checks if the response is clear, concise, and easy to understand. Example error types include: (1) Redundancy, (2) Irrelevance, (3) Overuse of jargon, (4) Lack of examples, (5) Lack of sources, etc.",
#     "Clarity":"",
#     "Fluency": "This aspect assesses the readability and understandability of the response. Errors in this stage could lead to answers that are difficult to understand, despite being factually correct and based on accurate information synthesis. Example error types include: (1) Unclear language, (2) Poor organization, (3) Lengthy responses, (4) Unexplained jargon, (5) Grammar or spelling errors, etc.",
# }
EVAL_ASPECTS["instruction-following"] = {
    "Comprehension": "This aspect evaluates how well the output understands the given instruction. Example error types include: (1) Misinterpretation, (2) Inability to grasp complex instructions, (3) Failure to understand context, etc.",
    "Accuracy": "This aspect measures the correctness of the output in relation to the instruction and the paired input context. Example error types include: (1) Inconsistent content with instruction and input, (2) Factual errors, (3) Misapplication of instructions, (4) Incorrect reasoning, etc.",
    "Informativeness": "This aspect assesses the relevancy and usefulness of the information provided by the output. Example error types include: (1) Irrelevant information, (2) Insufficient information, (3) Overload of information, (4) Outdated information, (5) Repetitive information, etc.",
    "Coherence": "This aspect evaluates how logically the output flows and connects. Errors in this stage could lead to answers that are difficult to understand, despite being factually correct and based on accurate information synthesis. Example error types include: (1) Grammar or spelling errors, (2) Poor organization, (3) Illogical sequence, (4) Inappropriate tone or style, (5) Non-sequiturs, etc.",
}

EVAL_ASPECTS["story_generation"] = {
    "Fluency": "How well the story is written in terms of grammar, spelling, punctuation, and coherence.",
    "Consistency": "How well the story is consistent in terms of plot, characters, setting, and themes.",
    "Style Matching": "How well the story matches the desired style or genre of the user or the task.",
    # "Interestingness": "How interesting the story is in terms of plot, characters, setting, and themes.", # maybe subjective
    # "Novelty": "How original and diverse the story is in terms of plot, characters, setting, and themes." # maybe delete
}

EVAL_ASPECTS["other"] = {
    "Consistency": "How well the generated text is consistent the input. It could be the consistency of the style, the topic, the genre, etc.",
    "Informativeness": "How well the generated text is informative.",
    "Rationality": "How well the generated text is rational. It should be consistent with the common sense.",
}

EVAL_ASPECTS["common_sense"] = {
    "Fact Accuracy": "Measured by the percentage of generated text that correctly states the facts,",
    "Relevance": "Measured by the percentage of generated text that is relevant to the input, such as concepts, entities, and relations",
    "Informativeness": "Measure by the percentage of generated facts in sentences that are helpful for humans to understand the data.",
}  # delay

EVAL_ASPECTS["image_captioning"] = {
    "Objects Accuracy": "The percentage of objects and their attributes in the image that are correctly mentioned in the caption.",
    "Actions Accuracy": "The percentage of actions performed by the objects and and relations between objects in the image that are correctly mentioned in the caption.",
    "Detail Richness": "Measured by the richness of the specific details for both the objects and actions in the image that are correctly mentioned in the caption.",
}  # not do this for now

tigerscore/xgptscore/constants.py
TIGER-AI-Lab/TIGERScore
import os
import copy
import functools
import json
import logging
import math
import multiprocessing
import random
import time
import hashlib
from typing import Optional, Sequence
from pathlib import Path
import numpy as np
import openai
import tiktoken
import tqdm

__all__ = ["openai_completions"]
tiktoken.model.MODEL_TO_ENCODING['ChatGPT'] = tiktoken.model.MODEL_TO_ENCODING['gpt-3.5-turbo']
### API specific ###
DEFAULT_OPENAI_API_BASE = openai.api_base
OPENAI_API_KEYS = os.environ.get("OPENAI_API_KEYS", os.environ.get("OPENAI_API_KEY", None))
if isinstance(OPENAI_API_KEYS, str):
    OPENAI_API_KEYS = OPENAI_API_KEYS.split(",")
OPENAI_ORGANIZATION_IDS = os.environ.get("OPENAI_ORGANIZATION_IDS", None)
if isinstance(OPENAI_ORGANIZATION_IDS, str):
    OPENAI_ORGANIZATION_IDS = OPENAI_ORGANIZATION_IDS.split(",")
cache_dir = Path(os.path.abspath(__file__)).parent / "cache"
cache_dir.mkdir(exist_ok=True)
cache_base_path = None
cache_base = None


def get_prompt_uids(prompt: str) -> str:
    return hashlib.sha256(prompt.encode()).hexdigest()

def openai_completions(
    prompts: Sequence[str],
    model_name: str,
    tokens_to_favor: Optional[Sequence[str]] = None,
    tokens_to_avoid: Optional[Sequence[str]] = None,
    is_skip_multi_tokens_to_avoid: bool = True,
    is_strip: bool = True,
    num_procs: Optional[int] = None,
    batch_size: Optional[int] = None,
    use_cache: bool = True,
    **decoding_kwargs,
) -> dict[str, list]:
    r"""Get openai completions for the given prompts. Allows additional parameters such as tokens to avoid and
    tokens to favor.

    Parameters
    ----------
    prompts : list of str
        Prompts to get completions for.

    model_name : str
        Name of the model to use for decoding.

    tokens_to_favor : list of str, optional
        Substrings to favor in the completions. We will add a positive bias to the logits of the tokens constituting
        the substrings.

    tokens_to_avoid : list of str, optional
        Substrings to avoid in the completions. We will add a large negative bias to the logits of the tokens
        constituting the substrings.

    is_skip_multi_tokens_to_avoid : bool, optional
        Whether to skip substrings from tokens_to_avoid that are constituted by more than one token => avoid undesired
        side effects on other tokens.

    is_strip : bool, optional
        Whether to strip trailing and leading spaces from the prompts.
    
    use_cache : bool, optional
        Whether to use cache to save the query results in case of multiple queries.

    decoding_kwargs :
        Additional kwargs to pass to `openai.Completion` or `openai.ChatCompletion`.

    Example
    -------
    >>> prompts = ["Respond with one digit: 1+1=", "Respond with one digit: 2+2="]
    >>> openai_completions(prompts, model_name="text-davinci-003", tokens_to_avoid=["2"," 2"])['completions']
    ['\n\nAnswer: \n\nTwo (or, alternatively, the number "two" or the numeral "two").', '\n\n4']
    >>> openai_completions(prompts, model_name="text-davinci-003", tokens_to_favor=["2"])['completions']
    ['2\n\n2', '\n\n4']
    >>> openai_completions(prompts, model_name="text-davinci-003",
    ... tokens_to_avoid=["2 a long sentence that is not a token"])['completions']
    ['\n\n2', '\n\n4']
    >>> chat_prompt = ["<|im_start|>user\n1+1=<|im_end|>", "<|im_start|>user\nRespond with one digit: 2+2=<|im_end|>"]
    >>> openai_completions(chat_prompt, "gpt-3.5-turbo", tokens_to_avoid=["2"," 2"])['completions']
    ['As an AI language model, I can confirm that 1+1 equals  02 in octal numeral system, 10 in decimal numeral
    system, and  02 in hexadecimal numeral system.', '4']
    """
    # add cache support for query

    if use_cache:
        global cache_base
        global cache_base_path
        cache_base_path = cache_dir / f"{model_name}.jsonl"
        if cache_base is None:
            if not cache_base_path.exists():
                cache_base = {}
                logging.warning(f"Cache file {cache_base_path} does not exist. Creating new cache.")
            else:
                with open(cache_base_path, "r") as f:
                    cache_base = [json.loads(line) for line in f.readlines()]
                cache_base = {item['uid']: item for item in cache_base}
                logging.warning(f"Loaded cache base from {cache_base_path}.")
    
    n_examples = len(prompts)
    if n_examples == 0:
        logging.warning("No samples to annotate.")
        return []
    else:
        logging.warning(f"Using `openai_completions` on {n_examples} prompts using {model_name}.")

    if tokens_to_avoid or tokens_to_favor:
        tokenizer = tiktoken.encoding_for_model(model_name)

        logit_bias = decoding_kwargs.get("logit_bias", {})
        if tokens_to_avoid is not None:
            for t in tokens_to_avoid:
                curr_tokens = tokenizer.encode(t)
                if len(curr_tokens) != 1 and is_skip_multi_tokens_to_avoid:
                    logging.warning(f"'{t}' has more than one token, skipping because `is_skip_multi_tokens_to_avoid`.")
                    continue
                for tok_id in curr_tokens:
                    logit_bias[tok_id] = -100  # avoids certain tokens

        if tokens_to_favor is not None:
            for t in tokens_to_favor:
                curr_tokens = tokenizer.encode(t)
                for tok_id in curr_tokens:
                    logit_bias[tok_id] = 7  # increase log prob of tokens to match

        decoding_kwargs["logit_bias"] = logit_bias

    if is_strip:
        prompts = [p.strip() for p in prompts]

    is_chat = decoding_kwargs.get("requires_chatml", _requires_chatml(model_name))
    if is_chat:
        # prompts = [_prompt_to_chatml(prompt) for prompt in prompts]
        num_procs = num_procs or 4
        batch_size = batch_size or 1

        if batch_size > 1:
            logging.warning("batch_size > 1 is not supported yet for chat models. Setting to 1")
            batch_size = 1

    else:
        num_procs = num_procs or 1
        batch_size = batch_size or 10

    logging.warning(f"Kwargs to completion: {decoding_kwargs}")
    n_batches = int(math.ceil(n_examples / batch_size))

    prompt_batches = [prompts[batch_id * batch_size: (batch_id + 1) * batch_size] for batch_id in range(n_batches)]

    if "azure" in openai.api_base:
        # Azure API uses engine instead of model
        kwargs = dict(n=1, engine=model_name, is_chat=is_chat, use_cache=use_cache, **decoding_kwargs)
    else:
        # OpenAI API uses model instead of engine
        kwargs = dict(n=1, model=model_name, is_chat=is_chat, use_cache=use_cache, **decoding_kwargs)
    logging.warning(f"Kwargs to completion: {kwargs}")

    with Timer() as t:
        if num_procs == 1:
            completions = [
                _openai_completion_helper(prompt_batch, **kwargs)
                for prompt_batch in tqdm.tqdm(prompt_batches, desc="prompt_batches")
            ]
        else:
            with multiprocessing.Pool(num_procs) as p:
                partial_completion_helper = functools.partial(_openai_completion_helper, **kwargs)
                completions = list(
                    tqdm.tqdm(
                        p.imap(partial_completion_helper, prompt_batches),
                        desc="prompt_batches",
                        total=len(prompt_batches),
                    )
                )
    logging.warning(f"Completed {n_examples} examples in {t}.")

    # flatten the list and select only the text
    completions_text = [completion['text'] for completion_batch in completions for completion in completion_batch]

    price = [
        completion["total_tokens"] * _get_price_per_token(model_name) 
        if completion["total_tokens"] is not None else 0
        for completion_batch in completions
        for completion in completion_batch
    ]
    avg_time = [t.duration / n_examples] * len(completions_text)

    return dict(completions=completions_text, price_per_example=price, time_per_example=avg_time)


def _openai_completion_helper(
    prompt_batch: Sequence[str],
    is_chat: bool,
    sleep_time: int = 2,
    openai_organization_ids: Optional[Sequence[str]] = OPENAI_ORGANIZATION_IDS,
    openai_api_keys: Optional[Sequence[str]] = OPENAI_API_KEYS,
    openai_api_base: Optional[str] = None,
    max_tokens: Optional[int] = 1000,
    top_p: Optional[float] = 1.0,
    temperature: Optional[float] = 0.7,
    use_cache: bool = True,
    **kwargs,
):
    # randomly select orgs
    if openai_organization_ids is not None:
        openai.organization = random.choice(openai_organization_ids)

    if openai_api_keys is not None:
        openai.api_key = random.choice(openai_api_keys)

    # set api base
    openai.api_base = openai_api_base if openai_api_base is not None else DEFAULT_OPENAI_API_BASE

    # copy shared_kwargs to avoid modifying it
    kwargs.update(dict(max_tokens=max_tokens, top_p=top_p, temperature=temperature))
    curr_kwargs = copy.deepcopy(kwargs)

    if use_cache:
        prompt_uids = [get_prompt_uids(prompt) for prompt in prompt_batch]
        cache_completions = [cache_base[prompt_uid]['completion'] if prompt_uid in cache_base else None for prompt_uid in prompt_uids]
        to_query_prompt_batch = [prompt for prompt, cache_completion in zip(prompt_batch, cache_completions) if cache_completion is None]
    else:
        to_query_prompt_batch = prompt_batch
    if is_chat:
        to_query_prompt_batch = [_prompt_to_chatml(prompt) for prompt in to_query_prompt_batch]
    
    # now_cand = ""
    # retry_times = 0
    if len(to_query_prompt_batch) != 0:
        while True:
            try:
                if is_chat:
                    completion_batch = openai.ChatCompletion.create(messages=to_query_prompt_batch[0], **curr_kwargs)

                    choices = completion_batch.choices
                    for choice in choices:
                        assert choice.message.role == "assistant"
                        if choice.message.content == "":
                            choice["text"] = " "  # annoying doesn't allow empty string
                        else:
                            choice["text"] = choice.message.content

                        if choice.message.get("function_call"):
                            # currently we only use function calls to get a JSON object
                            # => overwrite text with the JSON object. In the future, we could
                            # allow actual function calls
                            all_args = json.loads(choice.message.function_call.arguments)
                            assert len(all_args) == 1
                            choice["text"] = all_args[list(all_args.keys())[0]]

                else:
                    completion_batch = openai.Completion.create(prompt=to_query_prompt_batch, **curr_kwargs)
                    choices = completion_batch.choices

                for choice in choices:
                    choice["total_tokens"] = completion_batch.usage.total_tokens / len(prompt_batch)
                break
            except openai.error.OpenAIError as e:
                if "Please reduce your prompt" in str(e):
                    kwargs["max_tokens"] = int(kwargs["max_tokens"] * 0.8)
                    logging.warning(f"Reducing target length to {kwargs['max_tokens']}, Retrying...")
                    if kwargs["max_tokens"] == 0:
                        logging.exception("Prompt is already longer than max context length. Error:")
                        raise e
                else:
                    if "rate limit" in str(e).lower():
                        logging.warning("Hit request rate limit; retrying...")
                    else:
                        logging.warning(f"Unknown error {e}. \n It's likely a rate limit so we are retrying...")
                        logging.warning(prompt_batch)
                    # if now_cand == to_query_prompt_batch[0]:
                    #     retry_times += 1
                    #     if retry_times > 5:
                    #         logging.exception("Too many retries. Error:")
                    #         to_query_prompt_batch = to_query_prompt_batch[1:]
                    #         retry_times = 0
                    # else:
                    #     now_cand = to_query_prompt_batch[0]
                    #     retry_times = 0
                    if openai_organization_ids is not None and len(openai_organization_ids) > 1:
                        openai.organization = random.choice(
                            [o for o in openai_organization_ids if o != openai.organization]
                        )
                        logging.warning(f"Switching OAI organization.")
                    if openai_api_keys is not None and len(openai_api_keys) > 1:
                        openai.api_key = random.choice([o for o in openai_api_keys if o != openai.api_key])
                        logging.warning(f"Switching OAI API key.")
                    time.sleep(sleep_time)  # Annoying rate limit on requests.
    
    if use_cache:
        responses = []
        to_cache_items = []
        to_query_idx = 0
        for i in range(len(prompt_batch)):
            prompt_uid = prompt_uids[i]
            if cache_completions[i] is None:
                cache_base[prompt_uid] = dict(uid=prompt_uid, 
                    prompt=prompt_batch[i], completion=choices[to_query_idx]['text'],
                    top_p=top_p, temperature=temperature, max_tokens=max_tokens, total_tokens=choices[to_query_idx]['total_tokens'])
                to_cache_items.append(cache_base[prompt_uid])
                responses.append(dict(text=choices[to_query_idx]['text'], total_tokens=choices[to_query_idx]['total_tokens']))
                to_query_idx += 1
            else:
                responses.append(dict(text=cache_completions[i], total_tokens=None))
        assert to_query_idx == len(to_query_prompt_batch)
        # save cache items
        with open(cache_base_path, "a+") as f:
            for item in to_cache_items:
                f.write(json.dumps(item, ensure_ascii=False) + "\n")
    else:
        responses = choices
    return responses


def _requires_chatml(model: str) -> bool:
    """Whether a model requires the ChatML format."""
    # TODO: this should ideally be an OpenAI function... Maybe it already exists?
    return "turbo" in model or "gpt-4" in model or "chatgpt" in model.lower()


def _prompt_to_chatml(prompt: str, start_token: str = "<|im_start|>", end_token: str = "<|im_end|>"):
    r"""Convert a text prompt to ChatML formal

    Examples
    --------
    >>> prompt = (
    ... "<|im_start|>system\n"
    ... "You are a helpful assistant.\n<|im_end|>\n"
    ... "<|im_start|>system name=example_user\nKnock knock.\n<|im_end|>\n<|im_start|>system name=example_assistant\n"
    ... "Who's there?\n<|im_end|>\n<|im_start|>user\nOrange.\n<|im_end|>"
    ... )
    >>> print(prompt)
    <|im_start|>system
    You are a helpful assistant.
    <|im_end|>
    <|im_start|>system name=example_user
    Knock knock.
    <|im_end|>
    <|im_start|>system name=example_assistant
    Who's there?
    <|im_end|>
    <|im_start|>user
    Orange.
    <|im_end|>
    >>> _prompt_to_chatml(prompt)
    [{'content': 'You are a helpful assistant.', 'role': 'system'},
      {'content': 'Knock knock.', 'role': 'system', 'name': 'example_user'},
      {'content': "Who's there?", 'role': 'system', 'name': 'example_assistant'},
      {'content': 'Orange.', 'role': 'user'}]
    """
    prompt = prompt.strip()
    assert prompt.startswith(start_token)
    assert prompt.endswith(end_token)

    message = []
    for p in prompt.split("<|im_start|>")[1:]:
        newline_splitted = p.split("\n", 1)
        role = newline_splitted[0].strip()
        content = newline_splitted[1].split(end_token, 1)[0].strip()

        if role.startswith("system") and role != "system":
            # based on https://github.com/openai/openai-cookbook/blob/main/examples
            # /How_to_format_inputs_to_ChatGPT_models.ipynb
            # and https://github.com/openai/openai-python/blob/main/chatml.md it seems that system can specify a
            # dictionary of other args
            other_params = _string_to_dict(role.split("system", 1)[-1])
            role = "system"
        else:
            other_params = dict()

        message.append(dict(content=content, role=role, **other_params))

    return message

def _chatml_to_prompt(message: Sequence[dict], start_token: str = "<|im_start|>", end_token: str = "<|im_end|>"):
    r"""Convert a ChatML message to a text prompt

    Examples
    --------
    >>> message = [
    ...     {'content': 'You are a helpful assistant.', 'role': 'system'},
    ...     {'content': 'Knock knock.', 'role': 'system', 'name': 'example_user'},
    ...     {'content': "Who's there?", 'role': 'system', 'name': 'example_assistant'},
    ...     {'content': 'Orange.', 'role': 'user'}
    ... ]
    >>> _chatml_to_prompt(message)
    '<|im_start|>system\nYou are a helpful assistant.\n<|im_end|>\n<|im_start|>system name=example_user\nKnock knock.\n<|im_end|>\n<|im_start|>system name=example_assistant\nWho\'s there?\n<|im_end|>\n<|im_start|>user\nOrange.\n<|im_end|>'
    """
    prompt = ""
    for m in message:
        role = m["role"]
        name = m.get("name", None)
        if name is not None:
            role += f" name={name}"
        prompt += f"<|im_start|>{role}\n{m['content']}\n<|im_end|>\n"
    return prompt

def _string_to_dict(to_convert):
    r"""Converts a string with equal signs to dictionary. E.g.
    >>> _string_to_dict(" name=user university=stanford")
    {'name': 'user', 'university': 'stanford'}
    """
    return {s.split("=", 1)[0]: s.split("=", 1)[1] for s in to_convert.split(" ") if len(s) > 0}


def _get_price_per_token(model):
    """Returns the price per token for a given model"""
    if "gpt-4" in model:
        return (
            0.03 / 1000
        )  # that's not completely true because decoding is 0.06 but close enough given that most is context
    elif "gpt-3.5-turbo" in model.lower() or 'chatgpt' in model.lower():
        return 0.002 / 1000
    elif "text-davinci-003" in model:
        return 0.02 / 1000
    else:
        logging.warning(f"Unknown model {model} for computing price per token.")
        return np.nan

class Timer:
    """Timer context manager"""

    def __enter__(self):
        """Start a new timer as a context manager"""
        self.start = time.time()
        return self

    def __exit__(self, *args):
        """Stop the context manager timer"""
        self.end = time.time()
        self.duration = self.end - self.start

    def __str__(self):
        return f"{self.duration:.1f} seconds"

# # Example usage with ChatGPT Azure Model
# from openai_utils import openai_completions, _chatml_to_prompt
# prompts = ["Respond with one digit: 1+1=", "Respond with one digit: 2+2="]
# chatmls = [[{"role":"system","content":"You are an AI assistant that helps people find information."},
#             {"role":"user","content": prompt}] for prompt in prompts]
# chatml_prompts = [_chatml_to_prompt(chatml) for chatml in chatmls]
# print(chatml_prompts)
# openai_completions(prompts, model_name="ChatGPT")['completions']
tigerscore/xgptscore/openai_utils.py
TIGER-AI-Lab/TIGERScore
"""
This file contains the process functions for each chatgpt query mode.
Add your own process function for a mode here.
"""
import logging
import functools
import random
import numpy as np
from typing import List, Tuple, Dict, Any
from .templates import *
from .process_utils import *
from .constants import *
from string import Template
MODE_TEMPLATE_MAP = {}
MODE_PROCESS_MAP = {}

# the decorator for registering a process function
def process_register(name=None):
    def register_func(func, name=name):
        if not name:
            # default name is the function name without the suffix "_process"
            name = func.__name__
            name = name.replace("_process", "")
        MODE_PROCESS_MAP[name] = func
        logging.info(f"Register process function: {name}")
        return func
    return register_func

# First, develop a process function for your mode.
@process_register()
@process_register(name="default")
def ea_process(item: XPGTItem):
    """
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = Template(DEFAULT_SYSTEM_MESSAGE).substitute(task=item.task)
    prompt1 = Template(EA_TEMPLATE[0]).substitute(
        generation_instruction=item.instruction,
        input_context=item.input,
        reference_output=item.ref_output,
        hypothesis_output=item.hypo_output,
        task=item.task,
    )
    prompt2 = Template(EA_TEMPLATE[1]).substitute(
        aspect_descriptions="[{}]".format(",".join(EVAL_ASPECTS[item.task].keys()))
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None # no response to postprocess
    messages[1]['postprocess'] = default_postprocess
    messages[2]['postprocess'] = json_postprocess    
    return messages


@process_register()
def old_ea_process(item: XPGTItem):
    """
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = Template(CHATGPT_SYSTEM_MESSAGE).substitute(task=item.task)
    prompt1 = Template(OLD_EA_TEMPLATES[0]).substitute(
        generation_instruction=item.instruction,
        input_context=item.input,
        reference_output=item.ref_output,
        hypothesis_output=item.hypo_output,
        task=item.task,
    )
    prompt2 = Template(OLD_EA_TEMPLATES[1]).substitute(
        aspect_descriptions="[{}]".format(",".join(EVAL_ASPECTS[item.task].keys()))
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None # no response to postprocess
    messages[1]['postprocess'] = default_postprocess
    messages[2]['postprocess'] = json_postprocess    
    return messages

@process_register()
def wmt_mqm_process(item: XPGTItem):
    """
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = Template(DEFAULT_SYSTEM_MESSAGE).substitute(task=item.task)
    if isinstance(item.ref_output, list) and len(item.ref_output) > 1:
        refs_str = "\n".join(["Reference Translation {}: {}".format(i+1, ref) for i, ref in enumerate(item.ref_output)])
    elif isinstance(item.ref_output, list) and len(item.ref_output) == 1:
        refs_str = "Reference Translation: " + item.ref_output[0]
    elif isinstance(item.ref_output, list) and len(item.ref_output) == 0:
        refs_str = "No Reference Translation Provided"
    elif isinstance(item.ref_output, str):
        refs_str = "Reference Translation: " + item.ref_output
    prompt1 = Template(WMT_MQM_TEMPLATES[0]).substitute(
        generation_instruction=item.instruction,
        input_context=item.input,
        reference_output=refs_str,
        hypothesis_output=item.hypo_output,
        task=item.task,
    )
    prompt2 = Template(WMT_MQM_TEMPLATES[1]).substitute(
        aspect_descriptions="[{}]".format(",".join(EVAL_ASPECTS[item.task].keys()))
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None # no response to postprocess
    messages[1]['postprocess'] = default_postprocess
    messages[2]['postprocess'] = json_postprocess    
    return messages

@process_register()
def stars_process(item: XPGTItem):
    """
    A prompt template for Summarization Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = Template(DEFAULT_SYSTEM_MESSAGE).substitute(task=item.task)
    io_prompt = Template(STARS_TEMPLATE[0]).substitute(
        input_context=item.input,
        reference_output=item.ref_output,
        hypothesis_output=item.hypo_output,
    )
    prompt1 = Template(STARS_TEMPLATE[1]).substitute(
        task = item.task,
        aspects_descriptions="\n".join([f"- {k}: {v}" for k,v in EVAL_ASPECTS[item.task].items()]),
    )
    prompt2 = Template(STARS_TEMPLATE[2]).substitute(
        aspects_list=",".join(EVAL_ASPECTS[item.task].keys())
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = None
    messages[2]['postprocess'] = default_postprocess
    messages[3]['postprocess'] = json_postprocess   
    return messages

@process_register()
def multi_aspects_process(item: XPGTItem):
    """
    A prompt template for Multi Aspects Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = Template(CHATGPT_SYSTEM_MESSAGE).substitute(task=item.task)
    io_prompt = Template(MULTI_ASPECTS_TEMPLATE[0]).substitute(
        input_context=item.input,
        reference_output=zip_reference_string_non_task(item.ref_output),
        hypothesis_output=item.hypo_output,
    )
    prompt1 = Template(MULTI_ASPECTS_TEMPLATE[1]).substitute(
        task = item.task,
        # aspects_descriptions="\n".join([f"- {k}: {v}" for k,v in EVAL_ASPECTS[item.task].items()]),
        aspects_descriptions=", ".join(EVAL_ASPECTS[item.task].keys()),
        # aspects_descriptions = ", ".join(["\""+ i + "\"" for i in EVAL_ASPECTS[item.task].keys()]),
    )
    prompt2 = Template(MULTI_ASPECTS_TEMPLATE[2]).substitute(
        aspects_list=", ".join(EVAL_ASPECTS[item.task].keys())
        # aspects_list = ", ".join(["\""+ i + "\"" for i in EVAL_ASPECTS[item.task].keys()]),
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = None
    messages[2]['postprocess'] = default_postprocess
    messages[3]['postprocess'] = json_postprocess   
    return messages

@process_register()
def old_multi_aspects_process(item: XPGTItem):
    """
    A prompt template for Multi Aspects Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = Template(CHATGPT_SYSTEM_MESSAGE).substitute(task=item.task)
    io_prompt = Template(OLD_MULTI_ASPECTS_TEMPLATE[0]).substitute(
        input_context=item.input,
        reference_output=item.ref_output,
        hypothesis_output=item.hypo_output,
    )
    prompt1 = Template(OLD_MULTI_ASPECTS_TEMPLATE[1]).substitute(
        task = item.task,
        # aspects_descriptions="\n".join([f"- {k}: {v}" for k,v in EVAL_ASPECTS[item.task].items()]),
        aspects_descriptions = ", ".join(["\""+ i + "\"" for i in EVAL_ASPECTS[item.task].keys()]),
    )
    prompt2 = Template(OLD_MULTI_ASPECTS_TEMPLATE[2]).substitute(
        # aspects_list=", ".join(EVAL_ASPECTS[item.task].keys())
        aspects_list = ", ".join(["\""+ i + "\"" for i in EVAL_ASPECTS[item.task].keys()]),
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = None
    messages[2]['postprocess'] = default_postprocess
    messages[3]['postprocess'] = json_postprocess   
    return messages

@process_register()
def one_shot_process(item: XPGTItem):
    """
    A prompt template for Multi Aspects Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = Template(CHATGPT_SYSTEM_MESSAGE).substitute(task=item.task)
    one_shot_prompt = Template(ONE_SHOT_TEMPLATE[0]).substitute(
        task = item.task,
        input_context=item.input,
        task_instruction=item.instruction,
    )
    io_prompt = Template(ONE_SHOT_TEMPLATE[1]).substitute(
        input_context=item.input,
        reference_output=item.ref_output,
        hypothesis_output=item.hypo_output,
    )
    prompt1 = Template(ONE_SHOT_TEMPLATE[2]).substitute(
        task = item.task,
        # aspects_descriptions="\n".join([f"- {k}: {v}" for k,v in EVAL_ASPECTS[item.task].items()]),
        aspects_descriptions=", ".join(EVAL_ASPECTS[item.task].keys()),
    )
    prompt2 = Template(ONE_SHOT_TEMPLATE[3]).substitute(
        aspects_list=", ".join(EVAL_ASPECTS[item.task].keys())
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": one_shot_prompt, "do_query": True},
        {"role": "user", "content": io_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = default_postprocess
    messages[2]['postprocess'] = None
    messages[3]['postprocess'] = default_postprocess
    messages[4]['postprocess'] = json_postprocess   
    return messages

def zip_reference_string_non_task(reference):
    if isinstance(reference,str):
        return "Reference: "+reference
    if isinstance(reference,list):
        reference = list(set(reference))
        return "\n".join(["Reference {}: {}".format(i+1, ref) for i, ref in enumerate(reference)]).strip()
    raise TypeError("Reference is not a string or a list of strings")

def choose_only_one_reference(reference):
    if isinstance(reference,str):
        return reference
    if isinstance(reference,list):
        return sorted(list(set(reference)),key=len,reverse=True)[0]
    raise TypeError("Reference is not a string or a list of strings")

def d2t_task_instruction(instruction:str):
    return instruction.lower().replace("following","Source").replace("below","").strip(".").strip()

def zip_reference_string(reference,task:str):
    if isinstance(reference,str):
        return "Reference "+task+" Output: "+reference
    if isinstance(reference,list):
        return "\n".join(["Reference "+task+" Output {}: {}".format(i+1, ref) for i, ref in enumerate(reference)]).strip()
    raise TypeError("Reference is not a string or a list of strings")

# @process_register()
# def summarization_process(item: XPGTItem):
#     """
#     A prompt template for Multi Aspects Quality Estimation 
#     Args:
#         task: a task dict
#         inst (str): the instruction
#         input (str): the input context
#         ref_output (str): the reference output
#         hypo_output (str): the hypothesis output
#     Returns:
#         results (List[dict]):
#             List of prompt messages, each message contains the role and content, map_func and do_query.
#                 map_func: the function that map the current message content and the previous messages to the next message content
#                 do_query: whether to query the model for the current message
#     """
#     sys_prompt = Template(CHATGPT_SYSTEM_MESSAGE).substitute(task=item.task)
    
#     io_prompt = Template(SUMMARIZATION_TEMPLATE[0]).substitute(
#         input_context=item.input,
#         reference_output=zip_reference_string(item.ref_output,item.task),
#         hypothesis_output=item.hypo_output,
#         generation_instruction=item.instruction,
#     )
#     prompt1 = Template(SUMMARIZATION_TEMPLATE[1]).substitute(
#         task = item.task,
#         # aspects_descriptions="\n".join([f"- {k}: {v}" for k,v in EVAL_ASPECTS[item.task].items()]),
#         aspects_descriptions=", ".join(EVAL_ASPECTS[item.task].keys()),
#         # aspects_descriptions = ", ".join(["\""+ i + "\"" for i in EVAL_ASPECTS[item.task].keys()]),
#     )
#     prompt2 = Template(SUMMARIZATION_TEMPLATE[2]).substitute(
#         aspects_list=", ".join(EVAL_ASPECTS[item.task].keys())
#         # aspects_list = ", ".join(["\""+ i + "\"" for i in EVAL_ASPECTS[item.task].keys()]),
#     )
#     messages = [
#         {"role": "system", "content": sys_prompt, "do_query": False},
#         {"role": "user", "content": io_prompt, "do_query": False},
#         {"role": "user", "content": prompt1, "do_query": True},
#         {"role": "user", "content": prompt2, "do_query": True},
#     ]
#     for msg in messages:
#         msg['map_func'] = default_msg_map
#     messages[0]['postprocess'] = None
#     messages[1]['postprocess'] = None
#     messages[2]['postprocess'] = default_postprocess
#     messages[3]['postprocess'] = json_postprocess   
#     return messages

@process_register()
def align_score_process(item:XPGTItem):
    """
    A prompt template for Multi Aspects Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = Template(CHATGPT_SYSTEM_MESSAGE).substitute(task=item.task)
    io_prompt = Template(ALIGN_SCORE_TEMPLATE[0]).substitute(
        input_context=item.input,
        reference_output=item.ref_output,
        hypothesis_output=item.hypo_output,
    )
    prompt1 = Template(ALIGN_SCORE_TEMPLATE[1]).substitute(
        task=item.task,
        aspects_descriptions="\n".join([f"- {k}: {v}" for k,v in EVAL_ASPECTS[item.task].items()]).strip(),
        # aspects_descriptions=", ".join(EVAL_ASPECTS[item.task].keys()),
    )
    prompt2 = Template(ALIGN_SCORE_TEMPLATE[2]).substitute(
        aspects_list=", ".join(EVAL_ASPECTS[item.task].keys())
        # aspects_list = ", ".join(["\""+ i + "\"" for i in EVAL_ASPECTS[item.task].keys()]),
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = None
    messages[2]['postprocess'] = default_postprocess
    messages[3]['postprocess'] = json_postprocess
    return messages

@process_register()
def kb_process(item:XPGTItem):
    """
    A prompt template for Multi Aspects Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = "You are a helpful assistant to help user find information"
    aspects = EVAL_ASPECTS[item.task]
    shuffle_aspect_names = list(aspects.keys())
    random.shuffle(shuffle_aspect_names)
    selected_aspect_names = shuffle_aspect_names[:random.randint(1,len(aspects))]
    error_req_msg = ""
    
    total_num_major_errors = 0
    total_num_minor_errors = 0
    for aspect_name in selected_aspect_names:
        aspect_definition = aspects[aspect_name]
        num_major_errors = random.randint(1, 2)
        num_minor_errors = random.randint(0, 1)
        total_num_major_errors += num_major_errors
        total_num_minor_errors += num_minor_errors
        req_mag = "- contains {} major errors and {} minor errors for aspect {}".format(
            num_major_errors, num_minor_errors, aspect_name
        )
        error_req_msg += req_mag + "\n"
    error_req_msg += "Thus the total number of major errors is {} and the total number of minor errors is {} for all the error aspects above".format(
        total_num_major_errors, total_num_minor_errors
    )

    if isinstance(item.ref_output, list):
        ref_output = random.choice(item.ref_output)
    elif isinstance(item.ref_output, str):
        ref_output = item.ref_output
    else:
        raise TypeError("Reference output is not a string or a list of strings")
    io_prompt = Template(KB_TEMPLATES[0]).substitute(
        generation_instruction=item.instruction,
        input_context=item.input,
        reference_output=ref_output,
        error_requirements=error_req_msg,
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = json_postprocess
    return messages

@process_register()
def d2t_process(item:XPGTItem):
    """
    A prompt template for Multi Aspects Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = Template(CHATGPT_SYSTEM_MESSAGE).substitute(task=item.task)
    io_prompt = Template(D2T_TEMPLATE[0]).substitute(
        input_context=item.input,
        reference_output=zip_reference_string_non_task(item.ref_output),
        hypothesis_output=item.hypo_output,
        generation_instruction=d2t_task_instruction(item.instruction),
    )
    prompt1 = Template(D2T_TEMPLATE[1]).substitute(
        task=item.task,
        generation_instruction=d2t_task_instruction(item.instruction),
        # aspects_descriptions="\n".join([f"- {k}: {v}" for k,v in EVAL_ASPECTS[item.task].items()]).strip(),
        aspects_descriptions=", ".join(EVAL_ASPECTS[item.task].keys()),
    )
    prompt2 = Template(D2T_TEMPLATE[2]).substitute(
        aspects_list=", ".join(EVAL_ASPECTS[item.task].keys())
        # aspects_list = ", ".join(["\""+ i + "\"" for i in EVAL_ASPECTS[item.task].keys()]),
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = None
    messages[2]['postprocess'] = default_postprocess
    messages[3]['postprocess'] = json_postprocess
    return messages

@process_register()
def kb_txt_process(item:XPGTItem):
    """
    A prompt template for Multi Aspects Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = "You are a helpful assistant to help user find information"
    aspects = EVAL_ASPECTS[item.task]
    shuffle_aspect_names = list(aspects.keys())
    random.shuffle(shuffle_aspect_names)
    selected_aspect_names = shuffle_aspect_names[:random.randint(1,len(aspects))]
    error_req_msg = ""
    
    for aspect_name in selected_aspect_names:
        aspect_definition = aspects[aspect_name]
        req_mag = "- contains at least 1 error for aspect {}: {}".format(
            aspect_name, aspect_definition
        )
        error_req_msg += req_mag + "\n"

    if isinstance(item.ref_output, list):
        ref_output = random.choice(item.ref_output)
    elif isinstance(item.ref_output, str):
        ref_output = item.ref_output
    else:
        raise TypeError("Reference output is not a string or a list of strings")
    io_prompt = Template(KB_TXT_TEMPLATES[0]).substitute(
        generation_instruction=item.instruction,
        input_context=item.input,
        reference_output=ref_output,
        error_requirements=error_req_msg,
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = default_postprocess
    return messages

@process_register()
def instruction_process(item:XPGTItem):
    """
    A prompt template for Multi Aspects Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    sys_prompt = Template(CHATGPT_SYSTEM_MESSAGE).substitute(task=item.task)
    io_prompt = Template(INSTRUCTION_TEMPLATE[0]).substitute(
        input_context=item.input,
        generation_instruction=item.instruction,
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = default_postprocess
    return messages

def joint_instruction_and_source(input_context:str,generation_instruction:str):
    if (not input_context) or input_context.strip() == "":
        return "Source:"+generation_instruction
    else:
        return "Task Instruction:"+generation_instruction+"\nSource:"+input_context

@process_register()
def longform_qa_process(item:XPGTItem):
    """
    A prompt template for Multi Aspects Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    
    sys_prompt = Template(CHATGPT_SYSTEM_MESSAGE).substitute(task=item.task)
    io_prompt = Template(LONGFORM_QA_TEMPLATE[0]).substitute(
        generation_instruction_and_source = joint_instruction_and_source(item.input,item.instruction),
        input_context=item.input,
        reference_output=zip_reference_string_non_task(item.ref_output),
        hypothesis_output=item.hypo_output,
        generation_instruction=item.instruction,
    )
    prompt1 = Template(LONGFORM_QA_TEMPLATE[1]).substitute(
        task=item.task,
        generation_instruction=d2t_task_instruction(item.instruction),
        # aspects_descriptions="\n".join([f"- {k}: {v}" for k,v in EVAL_ASPECTS[item.task].items()]).strip(),
        aspects_descriptions=", ".join(EVAL_ASPECTS[item.task].keys()),
    )
    prompt2 = Template(LONGFORM_QA_TEMPLATE[2]).substitute(
        aspects_list=", ".join(EVAL_ASPECTS[item.task].keys())
        # aspects_list = ", ".join(["\""+ i + "\"" for i in EVAL_ASPECTS[item.task].keys()]),
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = None
    messages[2]['postprocess'] = default_postprocess
    messages[3]['postprocess'] = json_postprocess
    return messages

@process_register()
def instruction_following_process(item:XPGTItem):
    """
    A prompt template for Multi Aspects Quality Estimation 
    Args:
        task: a task dict
        inst (str): the instruction
        input (str): the input context
        ref_output (str): the reference output
        hypo_output (str): the hypothesis output
    Returns:
        results (List[dict]):
            List of prompt messages, each message contains the role and content, map_func and do_query.
                map_func: the function that map the current message content and the previous messages to the next message content
                do_query: whether to query the model for the current message
    """
    
    sys_prompt = Template(CHATGPT_SYSTEM_MESSAGE).substitute(task=item.task)
    io_prompt = Template(INSTRUCTION_FOLLOWING_TEMPLATE[0]).substitute(
        generation_instruction_and_source = joint_instruction_and_source(item.input,item.instruction),
        input_context=item.input,
        reference_output=zip_reference_string_non_task(item.ref_output),
        hypothesis_output=item.hypo_output,
        generation_instruction=item.instruction,
    )
    prompt1 = Template(INSTRUCTION_FOLLOWING_TEMPLATE[1]).substitute(
        task=item.task,
        generation_instruction=d2t_task_instruction(item.instruction),
        # aspects_descriptions="\n".join([f"- {k}: {v}" for k,v in EVAL_ASPECTS[item.task].items()]).strip(),
        aspects_descriptions=", ".join(EVAL_ASPECTS[item.task].keys()),
    )
    prompt2 = Template(INSTRUCTION_FOLLOWING_TEMPLATE[2]).substitute(
        aspects_list=", ".join(EVAL_ASPECTS[item.task].keys())
        # aspects_list = ", ".join(["\""+ i + "\"" for i in EVAL_ASPECTS[item.task].keys()]),
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = None
    messages[2]['postprocess'] = default_postprocess
    messages[3]['postprocess'] = json_postprocess
    return messages

@process_register()
def paraphrase_process(item:XPGTItem):
    sys_prompt = "You are a helpful assistant to help user find information"
    
    if isinstance(item.ref_output, list):
        ref_output = random.choice(item.ref_output)
    elif isinstance(item.ref_output, str):
        ref_output = item.ref_output
    else:
        raise TypeError("Reference output is not a string or a list of strings")
    io_prompt = Template(PARAPHRASE_TEMPLATES[0]).substitute(
        generation_instruction=item.instruction,
        input_context=item.input,
        reference_output=ref_output,
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": io_prompt, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = default_postprocess
    return messages

@process_register()
def mathqa_process(item:XPGTItem):
    sys_prompt = "You are a helpful assistant to help user find information"
    
    if isinstance(item.ref_output, list):
        ref_output = random.choice(item.ref_output)
    elif isinstance(item.ref_output, str):
        ref_output = item.ref_output
    else:
        raise TypeError("Reference output is not a string or a list of strings")

    prompt1 = Template(MATHQA_TEMPLATES[0]).substitute(
        task=item.task,
        generation_instruction=item.instruction.strip("\n "),
        input_context=item.input.strip("\n "),
        reference_output=ref_output.strip("\n "),
        hypothesis_output=item.hypo_output.strip("\n "),
        aspects_list="\n".join([f"{i+1}. {aspect_name}" for i, aspect_name in enumerate(EVAL_ASPECTS[item.task].keys())]),
    )
    prompt2 = Template(MATHQA_TEMPLATES[1]).substitute(
        aspects_list="[{}]".format(
            ", ".join([f'"{aspect_name}"' for i, aspect_name in enumerate(EVAL_ASPECTS[item.task].keys())])
        )
    )
    messages = [
        {"role": "system", "content": sys_prompt, "do_query": False},
        {"role": "user", "content": prompt1, "do_query": True},
        {"role": "user", "content": prompt2, "do_query": True},
    ]
    for msg in messages:
        msg['map_func'] = default_msg_map
    messages[0]['postprocess'] = None
    messages[1]['postprocess'] = default_postprocess
    messages[2]['postprocess'] = json_postprocess
    return messages
tigerscore/xgptscore/process.py
TIGER-AI-Lab/TIGERScore
import os
import json
import json5
import logging
from pathlib import Path
from dataclasses import dataclass, asdict
from transformers import AutoTokenizer
from tqdm import tqdm
from typing import List, Union
from itertools import chain
os.environ["TOKENIZERS_PARALLELISM"] = "false"
@dataclass
class XPGTItem():
    task: str
    instruction: str
    input: str
    ref_output: Union[str, List[str]]
    hypo_output: str

# Message map functions
def default_msg_map(cur_message:dict, messages:List[dict]):
    """ Map the text and old messages to the new messages for query
    Args:
        text (str): the prompt text
        messages (List[dict]): the messages list before this query
    Returns:
        prompt (str): the prompt text
    """
    new_messages = messages + [{
        "role": cur_message['role'],
        "content": cur_message['content']}
    ]
    return new_messages

# Postprocess functions
def default_postprocess(content:str):
    return content

def json_postprocess(content:str):
    try:
        # find the json content
        json_content = content[content.find("{"):content.rfind("}")+1]
        json_content = json.loads(json_content)
        return json_content
    except json.decoder.JSONDecodeError:
        try:
            json_content = json5.loads(json_content)
            return json_content
        except:
            return content

tokenizer = None
def truncate_texts(texts:Union[List[str], List[List[str]]], max_length:int=None):
    """
    Truncate the texts to the max length.
    Args:
        texts (List[str] or List[List[str]]): The list of texts.
        max_length (int): The max length.
    Returns:
        List[str]: The truncated texts.
    """
    if max_length is None:
        return texts
    if isinstance(texts[0], list) and \
        (
            all([len(x) == 0 for x in texts]) or \
            all([x is None for x in list(chain(*texts))])
        ) or isinstance(texts[0], str) and \
        all([x is None for x in list(chain(texts))]):
        logging.warning("All texts are None, skip truncating")
        return texts
    # using llama tokenizer by default
    global tokenizer
    disable_tqdm = len(texts) < 1000
    logging.warning(f"Truncating texts to max length {max_length}")
    if tokenizer is None:
        tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf", use_auth_token=True)
    # ...
    token_ids = []
    for text in tqdm(texts, desc="Truncating texts (tokenizing)", disable=disable_tqdm):
        if isinstance(text, list):
            token_ids.append([tokenizer.encode(x, add_special_tokens=False) for x in text])
        else:
            token_ids.append(tokenizer.encode(text, add_special_tokens=False))
    # ...
    truncated_texts = []
    for i, _token_ids in tqdm(enumerate(token_ids), desc="Truncating texts (truncating)", disable=disable_tqdm):
        if (len(_token_ids)) and isinstance(_token_ids[0], list):
            truncated_texts.append([])
            for _token_id in _token_ids:
                if len(_token_id) > max_length:
                    truncated_text = tokenizer.decode(_token_id[:max_length], skip_special_tokens=True)
                    truncated_text = truncated_text + " ..."
                else:
                    truncated_text = tokenizer.decode(_token_id, skip_special_tokens=True)
                truncated_texts[i].append(truncated_text)
        else:
            if len(_token_ids) > max_length:
                truncated_text = tokenizer.decode(_token_ids[:max_length], skip_special_tokens=True)
                truncated_text = truncated_text + " ..."
            else:
                truncated_text = tokenizer.decode(_token_ids, skip_special_tokens=True)
                
            truncated_texts.append(truncated_text)
    return truncated_texts
    
def truncate_items(items:List[XPGTItem], max_lengths):
    """
    Truncate the texts in the items to the max length.
    Args:
        items (List[XPGTItem]): The list of items.
        max_length (int): The max length.
    Returns:
        List[XPGTItem]: The truncated items.
    """
    truncated_inputs = truncate_texts([item.input for item in items], max_lengths.get("input", None))
    truncated_insts = truncate_texts([item.instruction for item in items], max_lengths.get("instruction", None))
    truncated_ref_outputs = truncate_texts([item.ref_output for item in items], max_lengths.get("ref_output", None))
    truncated_hypo_outputs = truncate_texts([item.hypo_output for item in items], max_lengths.get("hypo_output", None))
    for i, item in enumerate(items):
        item.instruction = truncated_insts[i]
        item.input = truncated_inputs[i]
        item.ref_output = truncated_ref_outputs[i]
        item.hypo_output = truncated_hypo_outputs[i]
    return items

def get_query_messages(messages: List[dict], queried_messages: List[dict]):
    """
    Args:
        messages (List[dict]): the messages list to add for query
        queried_messages (List[dict]): the messages list already queried, which contains the query responses also,
    Returns:
        new_messages (List[dict]): the new messages list to query
        postprocess (function): the postprocess function for the query response
    """
    if len(queried_messages) == 0:
        last_prompt_idx = -1
    else:
        assert len(queried_messages) >= 2, "queried_messages should have at least 2 messages, i.e., the user (system) and the response"
        last_prompt = queried_messages[-2]['content']
        prompt_texts = [x['content'] for x in messages]
        last_prompt_idx = prompt_texts.index(last_prompt)
    if last_prompt_idx == len(messages) - 1:
        return None
    new_messages = queried_messages.copy()
    for idx in range(last_prompt_idx+1, len(messages)):
        new_messages = messages[idx]["map_func"](messages[idx], new_messages)
        if messages[idx]["do_query"]:
            break
    return new_messages, messages[idx]["postprocess"]

def get_xgptscore_from_json(json_content: dict):
    """
    Args:
        json_content (dict): the json content
    Returns:
        xgptscore (float): the xgptscore, i.e. the sum of the reduction scores for all the errors
    """
    if isinstance(json_content, str):
        return None
    try:
        xgptscore = 0
        for error in json_content['errors'].values():
            if error['score_reduction'] == "N/A":
                continue
            xgptscore -= error['score_reduction']
        return xgptscore  
    except:
        return None


def get_xgptscore_from_json_star(json_content: dict):
    """
    Args:
        json_content (dict): the json content
    Returns:
        xgptscore (float): the xgptscore, i.e. the sum of the reduction scores for all the errors
    """
    xgptscore = 0
    res = {}
    for aspect_key, aspect in json_content.items():
        if isinstance(aspect,dict):
            score = aspect['Score']
            try:
                score = float(score)
            except:
                score = 0
            xgptscore += score
            res["xgptscore_"+aspect_key] = score
    res["xgptscore"] = xgptscore
    return res

def get_xgptscore_from_json_per_aspect(json_content: dict):
    """
    Args:
        json_content (dict): the json content
    Returns:
        xgptscore (float): the xgptscore, i.e. the sum of the reduction scores for all the errors
    """
    if not isinstance(json_content,dict):
        return None
    xgptscore = 0
    res = {}
    for error in json_content['errors'].values():
        if error['error_aspect'] is not None:
            if ("xgptscore_"+error['error_aspect'] not in res):
                res["xgptscore_"+error['error_aspect']] = 0
            res["xgptscore_"+error['error_aspect']] -= error['score_reduction']
            xgptscore -= error['score_reduction']
    res["xgptscore"] = xgptscore
    return res
tigerscore/xgptscore/process_utils.py
TIGER-AI-Lab/TIGERScore
DEFAULT_SYSTEM_MESSAGE = "You are an AI assistant to help user identify errors in a model generated output for the ${task} task."
CHATGPT_SYSTEM_MESSAGE = "You are an AI assistant."
EA_TEMPLATE = ["""\
Source: ${input_context}
Output: ${hypothesis_output}
The output is generated by a model with the following instruction: ${generation_instruction}
The correct output for this source input is: ${reference_output}
Identify the major and minor errors in this model generated output. 
Note that Major errors refer to actual errors that affects the task severely, and Minor errors refer to small imperfections, and purely subjective opinions about the output. \
You can compare the output with the reference output to identify the errors, but should not explain by simply mentioning the difference between them. \
Instead, give reasonable explanations like without a reference. Also explain like an expert in the task domain, not like a human annotator. \
"""
,
"""For each error you give in the response, please also elaborate the following information in a specific json format:
- error location (the substring that is wrong in the hypothesis output)
- error aspect it belongs to. Choose from ${aspect_descriptions}
- concise explanation why it's an error, and the correction suggestions. 
- severity of the error ("Major" or "Minor"). 
- reduction of score (Between 0.5 and 5 given the severity of the error)
Output JSON format:
- if there is no error
```
{"errors": {}}
```
- if there are errors
```
{"errors": { "error_1": { "error_location": "...", "error_aspect": "...", "explanation": "...", "severity": "...", "score_reduction": ..., }, ...}
```
""",                  
]

# Old WMT MQM templates, Do not delete this, archive only
OLD_EA_TEMPLATES = ["""\
Source: ${input_context}
Reference: ${reference_output}
Output: ${hypothesis_output}
Based on the given source and reference, identify the major and minor errors in this output for a(an) ${task} task. \
Note that Major errors refer to actual errors that affects the task severely, and Minor errors refer to smaller imperfections, and purely subjective opinions about the output.
"""
,
"""\
## For each error you give in the response, please also elaborate the following information:
- error location (the words that are wrong in the hypothesis output)
- error aspect it belongs to. Choose from ${aspect_descriptions}
- explanation why it's an error, and the correction suggestions. Do not mention the reference output in the explanation. Instead, explain from the perspective of the source and the hypothesis output.
- severity of the error ("Major" or "Minor"). 
- reduction of score (Major for 5, Minor for 1)
## Output JSON format:
- if there is no error
```
{"errors": {}}
```
- if there are errors
```
{"errors": { "error_1": { "error_location": "...", "error_aspect": "...", "explanation": "...", "severity": "...", "score_reduction": ..., }, ...}
```
"""                 
]

# This template's performance has been verified
WMT_MQM_TEMPLATES = ["""\
Translation Instruction: ${generation_instruction}
Source Text: ${input_context}
${reference_output}
Model-generated Translation: ${hypothesis_output}

Please identify and categorize the errors in the model-generated translation as either Major or Minor. Major errors significantly impact the task, while Minor errors are subjective and represent minor imperfections.

When identifying errors, do not solely rely on the reference translation for comparison. Provide explanations as an expert in the task domain, without explicitly mentioning the reference output.
""",
"""\
For each error identified above, please provide the following information in a specific JSON format:
- Error Location: The substring in the model-generated translation that contains the error.
- Error Aspect: Choose from ${aspect_descriptions}.
- Explanation: Describe why the identified issue is an error, and offer suggestions for correction. Explain as an expert in the task domain, without explicitly mentioning the reference output.
- Severity: Classify the error as "Major" or "Minor".
- Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

JSON Format for Output:
- If there are no errors:
```
{"errors": {}}
```
- If there are errors:
```
{"errors": {
"error_1": {
"error_location": "...",
"error_aspect": "...",
"explanation": "...",
"severity": "...",
"score_reduction": ...
},
...
}}
```
""",  # explanation, error_location, error_aspect, severity, score_reduction
]

STARS_TEMPLATE = ["""\
Source: ${input_context}
Reference: ${reference_output}
Output: ${hypothesis_output}\
""", # input_context, reference_output, hypothesis_output
"""\
Based on the given source and reference, identify the major and minor errors in this output for a(an) ${task} task. \
Note that Major errors refer to actual errors that affects the task severely, may change the meaning of the output, and Minor errors refer to smaller imperfections, and purely subjective opinions about the output. \
There may be multiple errors or no error in the output. Please give the error location.
Here are the aspects you need to check: 
${aspects_descriptions}
""", # task
"""\
## Please check your response, score the summarization to aspects below, with 0.5 to 5.

Here are the descriptions of the aspects:
${aspects_list}

## Output JSON format:
{
    "%Aspect%": {
        "Explanation": "...",
        "Score": "..."
    },
    ...
}
"""
]

MULTI_ASPECTS_TEMPLATE = [
"""\
Source: ${input_context}
Reference: ${reference_output}
Output: ${hypothesis_output}\
""", # input_context, reference_output, hypothesis_output
# """\
# Based on the given source and reference, identify the major and minor errors in this output for the ${task} task. \
# Major errors significantly impact the task or change the meaning, while Minor errors are subjective and represent minor imperfections. \
# There may be multiple errors or **no** error in the output. Here are the aspects you need to focus on: 
# ${aspects_descriptions}
# """, # task, aspects_descriptions  
"""\
Based on the given Source and Reference, please identify the major and minor errors in this Output for the summarization task. \
Major errors significantly impact the task or change the meaning, while Minor errors are subjective and represent minor imperfections. \
Here are the aspects you need to focus on: 
${aspects_descriptions}
There may be multiple errors or no error in the output.\
""", # task, aspects_descriptions  
# """\
# Based on the given source and reference, identify the major and minor errors in this output for the ${task} task. \
# Major errors significantly impact the task or change the meaning, while Minor errors are subjective and represent minor imperfections. \
# Here are the aspects you need to focus on: 
# ${aspects_descriptions}
# """, # task, aspects_descriptions  
# For long text, we need" There may be multiple errors or no error in the output."
# For short text, delete "There may be multiple errors or no error in the output."
"""For each error identified above, please provide the following information in a specific JSON format:
- Error Location: The substring in the hypothesis output that contains the error.
- Error Aspect: Choose only one from ${aspects_list}.
- Explanation: Describe why the identified issue is an error, and offer suggestions for correction. Explain as an expert in the task domain, without explicitly mentioning the reference output.
- Severity: Classify the error as "Major" or "Minor".
- Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

JSON Format for Output:
- If there are no errors:
```
{"errors": {}}
```
- If there are errors:
{"errors": {
"error_1": {
"error_location": "...",
"error_aspect": "...",
"explanation": "...",
"severity": "...",
"score_reduction": ...
},
...
}}\
""", # aspect_descriptions
]

ONE_SHOT_TEMPLATE = ["""\
Task instruction: ${task_instruction}\
Here is a input for the ${task} task:
${input_context}

Output:
""",
"""
Source: ${input_context}
Reference: ${reference_output}
Output: ${hypothesis_output}\
""",
"""\
Based on the given source and reference, identify the major and minor errors in this output for the ${task} task. \
Major errors significantly impact the task or change the meaning, while Minor errors are subjective and represent minor imperfections. \
There may be multiple errors or no error in the output. Here are the aspects you need to focus on: 
${aspects_descriptions}
""", # task, aspects_descriptions
"""For each error identified above, please provide the following information in a specific JSON format:
- Error Location: The substring in the hypothesis output that contains the error.
- Error Aspect: Choose only one from ${aspects_list}.
- Explanation: Describe why the identified issue is an error, and offer suggestions for correction. Explain as an expert in the task domain, without explicitly mentioning the reference output.
- Severity: Classify the error as "Major" or "Minor".
- Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

JSON Format for Output:
- If there are no errors:
```
{"errors": {}}
```
- If there are errors:
{"errors": {
"error_1": {
"error_location": "...",
"error_aspect": "...",
"explanation": "...",
"severity": "...",
"score_reduction": ...
},
...
}}\
""", # aspect_descriptions
]

OLD_MULTI_ASPECTS_TEMPLATE = [
"""\
Source: ${input_context}
Reference: ${reference_output}
Output: ${hypothesis_output}\
""", # input_context, reference_output, hypothesis_output
"""\
Based on the given source and reference, identify the major and minor errors in this output for the summarization task. \
Major errors significantly impact the task or change the meaning, while Minor errors are subjective and represent minor imperfections. \
Here are the aspects you need to focus on: 
${aspects_descriptions}
""", # task, aspects_descriptions  
# For long text, we need" There may be multiple errors or no error in the output."
# For short text, delete "There may be multiple errors or no error in the output."
"""For each error you give in the response, please also elaborate the following information:
- error location (the words that are wrong in the hypothesis output)
- error aspect it belongs to. Choose from aspects below.
- explanation why it's an error, and the correction suggestions. Do not mention the reference output in the explanation. Instead, explain from the perspective of the source and the hypothesis output.
- severity of the error (\"Major\" or \"Minor\").
- reduction of score (Between 0.5 and 5)

Here are the aspects you need to focus on:
${aspects_list}

## Output JSON format: 
# - if there is no error
# ```
# {\"errors\": {}}
# ```
# - if there are errors
# ```
# {\"errors\": { \"error_1\": { \"error_location\": \"...\", \"error_aspect\": \"...\", \"explanation\": \"...\", \"severity\": \"...\", \"score_reduction\": ..., }, ...}```\
""", # aspect_descriptions
]

# SUMMARIZATION_TEMPLATE = [
# """\
# Summarization Instruction:${generation_instruction}
# Source Text: ${input_context}
# ${reference_output}
# Model-generated Summary Output: ${hypothesis_output}\
# """, # input_context, reference_output, hypothesis_output
# """\
# Based on the given source and reference, identify the major and minor errors in this output for the ${task} task. \
# Major errors significantly impact the task or change the meaning, while Minor errors are subjective and represent minor imperfections. \
# There may be multiple errors or no error in the output. Here are the aspects you need to focus on: 
# ${aspects_descriptions}
# """, # task, aspects_descriptions  
# # """\
# # Based on the given source and reference, identify the major and minor errors in this output for the ${task} task. \
# # Major errors significantly impact the task or change the meaning, while Minor errors are subjective and represent minor imperfections. \
# # Here are the aspects you need to focus on: 
# # ${aspects_descriptions}
# # """, # task, aspects_descriptions  
# # For long text, we need" There may be multiple errors or no error in the output."
# # For short text, delete "There may be multiple errors or no error in the output."
# """For each error identified above, please provide the following information in a specific JSON format:
# - Error Location: The substring in the hypothesis output that contains the error.
# - Error Aspect: Choose only one from ${aspects_list}.
# - Explanation: Describe why the identified issue is an error, and offer suggestions for correction. Explain as an expert in the task domain, without explicitly mentioning the reference output.
# - Severity: Classify the error as "Major" or "Minor".
# - Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

# JSON Format for Output:
# - If there are no errors:
# ```
# {"errors": {}}
# ```
# - If there are errors:
# {"errors": {
# "error_1": {
# "error_location": "...",
# "error_aspect": "...",
# "explanation": "...",
# "severity": "...",
# "score_reduction": ...
# },
# ...
# }}\
# """, # aspect_descriptions
# ]

ALIGN_SCORE_TEMPLATE = [
"""\
Source: ${input_context}
Reference: ${reference_output}
Output: ${hypothesis_output}\
""", # input_context, reference_output, hypothesis_output
"""\
Based on the given Source and Reference, please evaluate the quality of summary(Output) written for the input text. \
Please score the summarization with 0.5 to 5 for aspects below. \
Then, identify the major and minor errors in this output for the ${task} task. \
There may be multiple errors or no error in the output. Here are the aspects you need to focus on: 
${aspects_descriptions}\
""", # Note that Major errors refer to actual errors that affects the task severely, may change the meaning of the output, and Minor errors refer to smaller imperfections, and purely subjective opinions about the output. \
"""For each error identified in your response, please provide the following information in a specific JSON format:
- Error Location: The substring in the Output that contains the error.
- Error Aspect: Choose only one from ${aspects_list}.
- Explanation: Describe why the identified issue is an error, and offer suggestions for correction. Explain as an expert in the task domain, without explicitly mentioning the reference output.
- Severity: Classify the error as "Major" or "Minor".
- Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

JSON Format for Output:
- If there are no errors:
```
{"errors": {}}
```
- If there are errors:
{"errors": {
"error_1": {
"error_location": "...",
"error_aspect": "...",
"explanation": "...",
"severity": "...",
"score_reduction": ...
},
...
}}\
""",
]

D2T_TEMPLATE = [
"""\
Task instruction:${generation_instruction}
Source: ${input_context}
${reference_output}
Output: ${hypothesis_output}\
""",
# """\
# Based on the given source and reference, identify the major and minor errors in this output for the data to text task.  \
# Note that Major errors refer to actual errors that affects the task severely, and Minor errors refer to smaller imperfections, and purely subjective opinions about the output. \
# Here are the aspects you need to focus on:
# ${aspects_descriptions}\
# """,
# """
# For each error you give in the response, please also elaborate the following information:
# - error location (the words that are wrong in the hypothesis output)
# - the main error aspect it belongs to. Choose **only one** from aspects below.
# - concise explanation why it's an error, and the correction suggestions. Do not mention the reference output in the explanation. Instead, explain from the perspective of the source and the hypothesis output.
# - severity of the error ("Major" or "Minor"). 
# - reduction of score (Between 0.5 and 5)

# Here are the aspects you need to focus on:
# ${aspects_descriptions}\

# ## Output JSON format:
# - if there is no error
# ```
# {"errors": {}}
# ```
# - if there are errors
# ```
# {"errors": { "error_1": { "error_location": "...", "error_aspect": "...", "explanation": "...", "severity": "...", "score_reduction": ..., }, ...}
# ```\
# """,
# """\
# Source: ${input_context}
# ${reference_output}
# Output: ${hypothesis_output}\
# """, # input_context, reference_output, hypothesis_output
# """\
# Based on the given Source and Reference, please evaluate the quality of Output written for the input text, which is to ${generation_instruction}. \
# Please score the Output with 0.5 to 5 for aspects below. \
# Then, identify the major and minor errors in this output for the ${task} task. \
# There may be multiple errors or no error in the output. Here are the aspects you need to focus on: 
# ${aspects_descriptions}\
# """,
# """For each error identified in your response, please provide the following information in a specific JSON format:
# - Error Location: The substring in the Output that contains the error.
# - Error Aspect: Choose only one from ${aspects_list}.
# - Explanation: Describe why the identified issue is an error, and offer suggestions for correction. Explain as an expert in the task domain, without explicitly mentioning the reference output.
# - Severity: Classify the error as "Major" or "Minor".
# - Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error and the score given.

# JSON Format for Output:
# - If there are no errors:
# ```
# {"errors": {}}
# ```
# - If there are errors:
# {"errors": {
# "error_1": {
# "error_location": "...",
# "error_aspect": "...",
# "explanation": "...",
# "severity": "...",
# "score_reduction": ...
# },
# ...
# }}\
# """,

"""\
Based on the given Source and reference, identify the major and minor errors in this Output for the data to text task, which is to ${generation_instruction}. \
Note that Major errors refer to actual errors that affects the task severely, may change the meaning of the output, and Minor errors refer to smaller imperfections, and purely subjective opinions about the output. \
There may be multiple errors or no error in the output.\
""", # aspects_list\  \  You should check about ${aspects_descriptions}. \
# """For each error identified in your response, please provide the following information in a specific JSON format:
# - Explanation: Describe why the identified issue is an error, and offer suggestions for correction. Explain as an expert in the task domain, without explicitly mentioning the reference output.
# - Error Location: The substring in the Output that contains the error.
# - Error Aspect: Choose only one from ${aspects_list}.
# - Severity: Classify the error as "Major" or "Minor".
# - Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

# JSON Format for Output:
# - If there are no errors:
# ```
# {"errors": {}}
# ```
# - If there are errors:
# {"errors": {
# "error_1": {
# "explanation": "...",
# "error_location": "...",
# "error_aspect": "...",
# "severity": "...",
# "score_reduction": ...
# },
# ...
# }}\
# """,
"""For each error identified in your response, please provide the following information in a specific JSON format:
- Error Location: The substring in the Output that contains the error.
- Error Aspect: Choose only one from ${aspects_list}.
- Explanation: Describe why the identified issue is an error, and offer suggestions for correction. Explain as an expert in the task domain, without explicitly mentioning the reference output.
- Severity: Classify the error as "Major" or "Minor".
- Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

JSON Format for Output:
- If there are no errors:
```
{"errors": {}}
```
- If there are errors:
{"errors": {
"error_1": {
"error_location": "...",
"error_aspect": "...",
"explanation": "...",
"severity": "...",
"score_reduction": ...
},
...
}}\
""",
]

KB_TEMPLATES = [
"""${generation_instruction}
Source Text: ${input_context}
Correct Output: ${reference_output}

Generate an incorrect output for the given source text. The incorrect output should satisfy the following requirements:
${error_requirements}
Note that major errors refer to actual errors that affects the task severely, and minor errors refer to smaller imperfections, and purely subjective opinions about the output. \
For each error, give me the error location, error aspect, explanation, severity ("major" or "minor"), and score reduction (between 0.5 to 5 given the severity of the error).
The output should be in the following format:
```
{
"incorrect_output": <your incorrect output>,
"errors": {
"error_1": {
"error_location": "...",
"error_aspect": "...",
"explanation": "...",
"severity": "...",
"score_reduction": ...
},
...
}
}
```
"""
]
KB_TXT_TEMPLATES = [
"""You are generating an output given the following instruction and context:
${generation_instruction}
${input_context}

The correct output is: ${reference_output}

Please generate an incorrect output for the given instruction and context by modifying the correct output. The incorrect output should contain the following errors:
${error_requirements}
For each error, give me the 
- error location (the substring that is wrong in the generated incorrect output)
- error aspect
- explanation (the generic error type description, why it's an error, and the correction suggestions)
- severity ("major" or "minor")
- score reduction (an integer between 1 to 5 given the severity of the error)

Output format:
Generated incorrect output: 

Error location 1:
Error aspect 1:
Explanation 1:
Severity 1:
Score reduction 1:
...
"""
]

INSTRUCTION_TEMPLATE = [
"""
${generation_instruction}

Source Text: ${input_context}
Output:\
"""
]

LONGFORM_QA_TEMPLATE = [
"""\
Source: ${input_context}
${reference_output}
Output: ${hypothesis_output}\
""",
"""\
Based on the given Source and reference, identify the major and minor errors in this Output for the ${task} task, which is to ${generation_instruction}. \
Note that Major errors refer to actual errors that affects the task severely, may change the meaning of the output, and Minor errors refer to smaller imperfections, and purely subjective opinions about the output. \
You should check about ${aspects_descriptions}.There may be multiple errors or no error in the output.\
""", # aspects_list\  \   \
# """For each error identified in your response, please provide the following information in a specific JSON format:
# - Explanation: Describe why the identified issue is an error, and offer suggestions for correction. Explain as an expert in the task domain, without explicitly mentioning the reference output.
# - Error Location: The substring in the Output that contains the error.
# - Error Aspect: Choose only one from ${aspects_list}.
# - Severity: Classify the error as "Major" or "Minor".
# - Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

# JSON Format for Output:
# - If there are no errors:
# ```
# {"errors": {}}
# ```
# - If there are errors:
# {"errors": {
# "error_1": {
# "explanation": "...",
# "error_location": "...",
# "error_aspect": "...",
# "severity": "...",
# "score_reduction": ...
# },
# ...
# }}\
# """
"""For each error identified in your response, please provide the following information in a specific JSON format:
- Error Location: The substring in the Output that contains the error.
- Error Aspect: Choose only one from ${aspects_list}.
- Explanation: Describe why the identified issue is an error, and offer suggestions for correction. Explain as an expert in the task domain, without explicitly mentioning the reference output.
- Severity: Classify the error as "Major" or "Minor".
- Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

JSON Format for Output:
- If there are no errors:
```
{"errors": {}}
```
- If there are errors:
{"errors": {
"error_1": {
"error_location": "...",
"error_aspect": "...",
"explanation": "...",
"severity": "...",
"score_reduction": ...
},
...
}}\
"""]

INSTRUCTION_FOLLOWING_TEMPLATE = [
"""\
${generation_instruction_and_source}
${reference_output}
Output: ${hypothesis_output}\
""",
"""\
Based on the given Source and reference, identify the major and minor errors in this Output for the ${task} task. \
Note that Major errors refer to actual errors that affects the task severely, may change the meaning of the output, and Minor errors refer to smaller imperfections, and purely subjective opinions about the output. \
You should check about ${aspects_descriptions}.There may be multiple errors or no error in the output.\
""", # aspects_list\  \   \
"""For each error identified in your response, please provide the following information in a specific JSON format:
- Error Location: The substring in the Output that contains the error.
- Error Aspect: Choose only one from ${aspects_list}.
- Explanation: Describe why the identified issue is an error, and offer suggestions for correction.
- Severity: Classify the error as "Major" or "Minor".
- Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

JSON Format for Output:
- If there are no errors:
```
{"errors": {}}
```
- If there are errors:
{"errors": {
"error_1": {
"error_location": "...",
"error_aspect": "...",
"explanation": "...",
"severity": "...",
"score_reduction": ...
},
...
}}\
"""]

PARAPHRASE_TEMPLATES = [
"""\
You are generating an output given the following instruction and context:
${generation_instruction}
${input_context}

An reference output is: 
${reference_output}

However, there might still be errors in the reference output. Please paraphrase the reference output to generate a new output that is better than the reference output. The new output should be grammatically correct and semantically consistent with the input. The new output should also be more fluent and natural than the reference output.
""",]


MATHQA_TEMPLATES = [
"""\
${generation_instruction}
${input_context}

The correct solution is:
${reference_output}

A model-generated solution is:
${hypothesis_output}

Please identify all the errors in this output considering the following aspects:
${aspects_list}
""",
"""
For each error identified in your response, please elaborate the following information:
- Error Location: The substring in the Output that contains the error.
- Error Aspect: Choose only one from ${aspects_list}
- Explanation: Describe why it is an error , and offer suggestions for correction.
- Severity: Classify the error as "Major" or "Minor".
- Score Reduction: Assign a reduction score between 0.5 and 5, considering the severity of the error.

JSON Format for Output:
- If there are no errors:
```
{"errors": {}}
```
- If there are errors:
{"errors": {
"error_1": {
"error_location": "...",
"error_aspect": "...",
"explanation": "...",
"severity": "...",
"score_reduction": ...
},
...
}}
"""
]
tigerscore/xgptscore/templates.py
TIGER-AI-Lab/TIGERScore
import os
import json
import logging
from .process import MODE_PROCESS_MAP
from .process_utils import XPGTItem, truncate_items, get_query_messages
from .openai_utils import openai_completions, _chatml_to_prompt
from typing import List, Tuple, Dict, Any, Union
from dataclasses import dataclass, asdict
from dacite import from_dict
from pathlib import Path
from functools import partial
def xgptscore(
    items: List[Union[XPGTItem, dict]],
    mode: str,
    model_name: str,
    num_workers: int=None,
    batch_size: int=None,
    **kwargs,
):
    config_path = os.path.join(os.path.dirname(__file__), f"mode_configs/{mode}.json")
    config_path = Path(config_path)
    if not config_path.exists():
        logging.warning(f"Config file {config_path} does not exist. Use default config.")
        config_path = config_path.with_name("default.json")
    
    with open(config_path, "r") as f:
        config = json.load(f)
    config.update(kwargs)
    if "max_lengths" in config:
        items = truncate_items(items, config["max_lengths"])

    if isinstance(items[0], dict):
        items = [from_dict(data_class=XPGTItem, data=item) for item in items]
    process_func = MODE_PROCESS_MAP[mode]
    if "process_kwargs" in config:
        process_func = partial(process_func, **config["process_kwargs"])
    process_results = list(map(process_func, items))

    total_round = len([x for x in process_results[0] if x['do_query']])
    logging.warning(f"Total chat rounds: {total_round}")
    logging.warning(f"Total chat messages: {len(items)}")
    # query and process
    round = 0
    queried_messages = [[] for _ in range(len(items))]
    total_price = 0
    total_time = 0
    round_completions = []
    while True:
        round += 1
        logging.warning(f"Processing chat round {round}/{total_round}")
        query_messages = list(map(get_query_messages, process_results, queried_messages))
        query_messages, postprocess_funcs = list(zip(*query_messages))
        chatml_prompts = list(map(_chatml_to_prompt, query_messages))
        openai_results = openai_completions(
            chatml_prompts,
            model_name=model_name,
            num_procs=num_workers,
            batch_size=batch_size,
            **config['decoding'],
        )
        completions = openai_results['completions']
        total_price += sum(openai_results['price_per_example'])
        total_time += sum(openai_results['time_per_example'])
        logging.warning(f"Round {round} price: {total_price}$")
        logging.warning(f"Round {round} time: {total_time}")
        postprocess_completions = [postprocess_funcs[idx](completion) for idx, completion in enumerate(completions)]
        round_completions.append(postprocess_completions)
        for idx, completion in enumerate(completions):
            queried_messages[idx] = query_messages[idx] + \
                [{"role": "assistant", "content": completion}] # add the assistant response
        if round == total_round:
            _query_messages = list(map(get_query_messages, process_results, queried_messages))
            assert all([x is None for x in _query_messages]), "All messages should be queried"
            break
    logging.warning(f"Total price: {total_price}$")
    logging.warning(f"Total time: {total_time}")
    logging.warning(f"Total time per example: {total_time / len(items)}")
    round_completions = list(zip(*round_completions))
    return dict(
        round_completions=round_completions,
        messages_records=queried_messages,
    )
    

        
"""
Example Usage:
task = "translation"
with open("example.json", "r") as f:
    items = json.load(f)
xgptitems = []
for item in items:
    for cand in item['candidates']:
        xgptitems.append(XPGTItem(
            task=task,
            instruction=item['instruction'],
            input=item['input'],
            ref_output=item['output'],
            hypo_output=cand['text']
        ))
result = xgptscore(xgptitems, "ea", "ChatGPT")
idx = 0
for item in items:
    for cand in item['candidates']:
        cand['responses'] = result['round_completions'][idx]
        cand['messages_records'] = result['messages_records'][idx]
json.dump(items, open("example_result.json", "w"), indent=4, ensure_ascii=False)
"""
tigerscore/xgptscore/xgptscore.py
TIGER-AI-Lab/TIGERScore