GPT-Fathom is an open-source and reproducible LLM evaluation suite, benchmarking 10+ leading open-source and closed-source LLMs as well as OpenAI's earlier models on 20+ curated benchmarks under aligned settings.
from .api import CompletionFn as CompletionFn
from .api import CompletionResult as CompletionResult
from .api import DummyCompletionFn as DummyCompletionFn
from .api import record_and_check_match as record_and_check_match
from .completion_fns.openai import AzureChatCompletionFn as AzureChatCompletionFn
from .completion_fns.openai import AzureCompletionFn as AzureCompletionFn
from .completion_fns.openai import OpenAIChatCompletionFn as OpenAIChatCompletionFn
from .completion_fns.openai import OpenAICompletionFn as OpenAICompletionFn
from .completion_fns.openai import OpenAICompletionResult as OpenAICompletionResult
from .data import get_csv as get_csv
from .data import get_json as get_json
from .data import get_jsonl as get_jsonl
from .data import get_jsonls as get_jsonls
from .data import get_lines as get_lines
from .data import iter_jsonls as iter_jsonls
from .eval import Eval as Eval

evals/__init__.py
GPT-Fathom/GPT-Fathom
"""
This file provides common interfaces and utilities used by eval creators to
sample from models and process the results.
"""

import logging
from abc import ABC, abstractmethod
from typing import Any, Callable, Optional, Protocol, Union, runtime_checkable

from evals.prompt.base import OpenAICreateChatPrompt, OpenAICreatePrompt, Prompt
from evals.record import record_match

logger = logging.getLogger(__name__)


class CompletionResult(ABC):
    @abstractmethod
    def get_completions(self) -> list[str]:
        pass


@runtime_checkable
class CompletionFn(Protocol):
    def __call__(
        self,
        prompt: Union[str, OpenAICreateChatPrompt],
        **kwargs,
    ) -> CompletionResult:
        """
        ARGS
        ====
        `prompt`: Either a `Prompt` object or a raw prompt that will get wrapped in
            the appropriate `Prompt` class.
        `kwargs`: Other arguments passed to the API.

        RETURNS
        =======
        The result of the API call.
        The prompt that was fed into the API call as a str.
        """


class DummyCompletionResult(CompletionResult):
    def get_completions(self) -> list[str]:
        return ["This is a dummy response."]


class DummyCompletionFn(CompletionFn):
    def __call__(
        self,
        prompt: Union[OpenAICreatePrompt, OpenAICreateChatPrompt, Prompt],
        **kwargs,
    ) -> CompletionResult:
        return DummyCompletionResult()


def record_and_check_match(
    prompt: Any,
    sampled: str,
    expected: Union[str, list[str], tuple[str]],
    separator: Callable[[str], bool] = None,
    options: Optional[list[str]] = None,
    detected: bool = False,
):
    """
    Records and checks if a sampled response from a CompletionFn matches the expected result.

    Args:
        prompt: The input prompt.
        sampled: The sampled response from the model.
        expected: The expected response or list of responses.
        separator: Optional function to check if a character is a separator.
        options: Optional list of options to match against the sampled response.

    Returns:
        The matched option or None if no match found.
    """
    if isinstance(expected, tuple):
        expected = list(expected)
    elif not isinstance(expected, list):
        expected = [expected]
    if options is None:
        options = expected

    picked = None
    for option in options:
        if not sampled.startswith(option):
            continue
        if (
            separator is not None
            and len(sampled) > len(option)
            and not separator(sampled[len(option)])
        ):
            continue
        picked = option
        break

    result = {
        "prompt": prompt,
        "sampled": sampled,
        "options": options,
        "picked": picked,
    }
    match = picked in expected
    result["expected"] = expected
    result["match"] = match
    record_match(
        match,
        expected=expected,
        picked=picked,
        sampled=sampled,
        options=options,
        detected=detected,
    )
    return picked

evals/api.py
GPT-Fathom/GPT-Fathom
"""
This file defines the base specifications for models, evals, and runs. Running
evals and most development work should not require familiarity with this file.
"""
import base64
import datetime
import os
from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence

if TYPE_CHECKING:
    from dataclasses import dataclass
else:
    from pydantic.dataclasses import dataclass


@dataclass
class CompletionFnSpec:
    """
    Specification for a CompletionFn.
    """

    cls: str
    args: Optional[Dict[str, Any]] = None
    key: Optional[str] = None
    group: Optional[str] = None


@dataclass
class BaseEvalSpec:
    """
    Specification for a base eval.
    """

    id: Optional[str] = None
    metrics: Optional[Sequence[str]] = None
    description: Optional[str] = None
    disclaimer: Optional[str] = None

    """
    True if higher values are better, False if lower values are better.
    This should really be part of a metric, but it's easier to put it here.
    """
    higher_is_better: bool = True

    key: Optional[str] = None
    group: Optional[str] = None


@dataclass
class EvalSpec:
    """
    Specification for an eval.
    """

    cls: str
    args: Optional[Dict[str, Any]] = None
    key: Optional[str] = None
    group: Optional[str] = None


@dataclass
class EvalSetSpec:
    """
    Specification for an eval set.
    """

    evals: Sequence[str]
    key: Optional[str] = None
    group: Optional[str] = None


@dataclass
class RunSpec:
    completion_fns: list[str]
    eval_name: str
    base_eval: str
    split: str
    run_config: Dict[str, Any]
    created_by: str
    run_id: str = None
    created_at: str = None
    setting_name: str = None

    def __post_init__(self):
        now = datetime.datetime.utcnow()
        rand_suffix = base64.b32encode(os.urandom(5)).decode("ascii")
        self.run_id = now.strftime("%y%m%d%H%M%S") + rand_suffix
        self.created_at = str(now)

evals/base.py
GPT-Fathom/GPT-Fathom
import json

from datasets import load_dataset

MC_ls = [
    "sat-math",
    "sat-en",
    "aqua-rat",
    "lsat-ar",
    "lsat-lr",
    "lsat-rc",
    "logiqa-en",
    "sat-en-without-passage",
]
Cloze_ls = ["math"]
import os

registry_path = os.path.join("..", "registry")


eval_yaml = ""
for item in MC_ls:
    id_name = item
    # print(file_name, id_name)
    cur = f"""
agieval-{id_name}-0shot:
  id: agieval-{id_name}.val.ab-v1
  metrics: [accuracy]
agieval-{id_name}.val.ab-v1:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    no_MC_prompt: True
    num_few_shot: 0
    max_tokens: 50
    type: MC

agieval-{id_name}-5shot:
  id: agieval-{id_name}.val.ab-v2
  metrics: [accuracy]
agieval-{id_name}.val.ab-v2:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    num_few_shot: 5
    max_tokens: 50
    type: MC

agieval-{id_name}-CoT:
  id: agieval-{id_name}.val.ab-v3
  metrics: [accuracy]
agieval-{id_name}.val.ab-v3:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    num_few_shot: 5
    CoT: True
    type: MC

"""
    eval_yaml += cur


for item in Cloze_ls:
    id_name = item
    # print(file_name, id_name)
    cur = f"""
agieval-{id_name}-0shot:
  id: agieval-{id_name}.val.ab-v1
  metrics: [accuracy]
agieval-{id_name}.val.ab-v1:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    no_MC_prompt: True
    num_few_shot: 0
    max_tokens: 50
    type: Cloze

agieval-{id_name}-5shot:
  id: agieval-{id_name}.val.ab-v2
  metrics: [accuracy]
agieval-{id_name}.val.ab-v2:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    num_few_shot: 5
    max_tokens: 50
    type: Cloze

agieval-{id_name}-CoT:
  id: agieval-{id_name}.val.ab-v3
  metrics: [accuracy]
agieval-{id_name}.val.ab-v3:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    num_few_shot: 5
    CoT: True
    type: Cloze
    
"""
    eval_yaml += cur

###############  Now for Chinese
MC_ls = [
    "gaokao-chinese",
    "gaokao-geography",
    "gaokao-history",
    "gaokao-biology",
    "gaokao-chemistry",
    "gaokao-english",
    "logiqa-zh",
    "gaokao-mathqa",
]
Cloze_ls = ["gaokao-mathcloze"]
IMC_ls = ["jec-qa-kd", "jec-qa-ca", "gaokao-physics"]
import os

registry_path = os.path.join("..", "registry")


for item in MC_ls:
    id_name = item
    # print(file_name, id_name)
    cur = f"""
agieval-{id_name}-0shot:
  id: agieval-{id_name}.val.ab-v1
  metrics: [accuracy]
agieval-{id_name}.val.ab-v1:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    no_MC_prompt: True
    num_few_shot: 0
    type: MC
    max_tokens: 50
    Cn: True

agieval-{id_name}-5shot:
  id: agieval-{id_name}.val.ab-v2
  metrics: [accuracy]
agieval-{id_name}.val.ab-v2:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    num_few_shot: 5
    max_tokens: 50
    Cn: True
    type: MC

agieval-{id_name}-CoT:
  id: agieval-{id_name}.val.ab-v3
  metrics: [accuracy]
agieval-{id_name}.val.ab-v3:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    num_few_shot: 5
    Cn: True
    CoT: True
    type: MC

"""
    eval_yaml += cur

for item in IMC_ls:
    id_name = item
    # print(file_name, id_name)
    cur = f"""
agieval-{id_name}-0shot:
  id: agieval-{id_name}.val.ab-v1
  metrics: [accuracy]
agieval-{id_name}.val.ab-v1:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    no_MC_prompt: True
    Cn: True
    max_tokens: 50
    num_few_shot: 0
    type: IMC

agieval-{id_name}-5shot:
  id: agieval-{id_name}.val.ab-v2
  metrics: [accuracy]
agieval-{id_name}.val.ab-v2:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    num_few_shot: 5
    max_tokens: 50
    Cn: True
    type: IMC

agieval-{id_name}-CoT:
  id: agieval-{id_name}.val.ab-v3
  metrics: [accuracy]
agieval-{id_name}.val.ab-v3:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    num_few_shot: 5
    Cn: True
    CoT: True
    type: IMC
    
"""
    eval_yaml += cur


for item in Cloze_ls:
    id_name = item
    # print(file_name, id_name)
    cur = f"""
agieval-{id_name}-0shot:
  id: agieval-{id_name}.val.ab-v1
  metrics: [accuracy]
agieval-{id_name}.val.ab-v1:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    no_MC_prompt: True
    num_few_shot: 0
    max_tokens: 50
    type: Cloze
    Cn: True

agieval-{id_name}-5shot:
  id: agieval-{id_name}.val.ab-v2
  metrics: [accuracy]
agieval-{id_name}.val.ab-v2:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    num_few_shot: 5
    max_tokens: 50
    type: Cloze
    Cn: True

agieval-{id_name}-CoT:
  id: agieval-{id_name}.val.ab-v3
  metrics: [accuracy]
agieval-{id_name}.val.ab-v3:
  class: evals.elsuite.dataset_specific.agieval:AGIEVAL
  args:
    sample_jsonl: agieval/{id_name}.jsonl
    task: {id_name}
    few_shot_jsonl: agieval/few_shot_prompts.csv
    num_few_shot: 5
    Cn: True
    CoT: True
    type: Cloze
    
"""
    eval_yaml += cur


with open(os.path.join(registry_path, "evals", "agieval.yaml"), "w") as f:
    f.write(eval_yaml)

evals/build_data/build_agieval.py
GPT-Fathom/GPT-Fathom
"""Build the relevant file for lambada dataset"""
import json
import os

from datasets import load_dataset


def create_chat_prompt(text):
    return [
        {
            "role": "system",
            "content": "Please answer with the word which is most likely to follow:",
        },
        {"role": "user", "content": text},
    ]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


# Build the test samples for arc-easy
samples = load_dataset("ai2_arc", "ARC-Easy", split="test")

registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "arc"), exist_ok=True)

data = []

for item in samples:
    pair = {
        "question": item["question"],
        "choices": item["choices"]["text"],
        "answer": item["choices"]["label"].index(item["answerKey"]),
    }
    data.append(pair)

file_path = os.path.join(registry_path, "data/arc/samples_easy.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")


# Build the test samples for arc-challenge
samples = load_dataset("ai2_arc", "ARC-Challenge", split="test")

registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "arc"), exist_ok=True)

data = []

for item in samples:
    pair = {
        "question": item["question"],
        "choices": item["choices"]["text"],
        "answer": item["choices"]["label"].index(item["answerKey"]),
    }
    data.append(pair)

file_path = os.path.join(registry_path, "data/arc/samples_challenge.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")


# Build the few shot samples for Easy

import os

registry_path = os.path.join("..", "registry")
samples = load_dataset("ai2_arc", "ARC-Easy", split="validation")
os.makedirs(os.path.join(registry_path, "data", "arc"), exist_ok=True)

data = []
for item in samples:
    pair = {
        "question": item["question"],
        "choices": item["choices"]["text"],
        "answer": item["choices"]["label"].index(item["answerKey"]),
    }
    data.append(pair)
data = data[:50]
file_path = os.path.join(registry_path, "data/arc/fewshot_easy.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

# Build the few shot samples for Challenge

import os

registry_path = os.path.join("..", "registry")
samples = load_dataset("ai2_arc", "ARC-Challenge", split="validation")
os.makedirs(os.path.join(registry_path, "data", "arc"), exist_ok=True)

data = []
for item in samples:
    pair = {
        "question": item["question"],
        "choices": item["choices"]["text"],
        "answer": item["choices"]["label"].index(item["answerKey"]),
    }
    data.append(pair)
data = data[:50]
file_path = os.path.join(registry_path, "data/arc/fewshot_challenge.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

evals/build_data/build_arc.py
GPT-Fathom/GPT-Fathom
import json

from datasets import load_dataset

name = [
    "boolean_expressions",
    "causal_judgement",
    "date_understanding",
    "disambiguation_qa",
    "dyck_languages",
    "formal_fallacies",
    "geometric_shapes",
    "hyperbaton",
    "logical_deduction_five_objects",
    "logical_deduction_seven_objects",
    "logical_deduction_three_objects",
    "movie_recommendation",
    "multistep_arithmetic_two",
    "navigate",
    "object_counting",
    "penguins_in_a_table",
    "reasoning_about_colored_objects",
    "ruin_names",
    "salient_translation_error_detection",
    "snarks",
    "sports_understanding",
    "temporal_sequences",
    "tracking_shuffled_objects_five_objects",
    "tracking_shuffled_objects_seven_objects",
    "tracking_shuffled_objects_three_objects",
    "web_of_lies",
    "word_sorting",
]


def create_chat_prompt(text, instruction):
    return [
        {"role": "system", "content": instruction},
        {"role": "user", "content": text},
    ]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


import os

registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "bbh"), exist_ok=True)

# Build test samples
eval_yaml = ""
for task in name:
    data = []
    file_path = os.path.join(registry_path, f"data/bbh/samples_{task}.jsonl")
    samples = load_dataset("lukaemon/bbh", task, split="test")
    for sample in samples:
        pair = {"input": sample["input"], "ideal": sample["target"]}
        data.append(pair)

    print(file_path)
    with open(file_path, "w") as f:
        for entry in data:
            json.dump(entry, f)
            f.write("\n")

        cur = f"""
bbh-{task}-3shot:
  id: bbh-{task}.val.ab-v1
  metrics: [accuracy]
bbh-{task}.val.ab-v1:
  class: evals.elsuite.dataset_specific.bbh:BBH
  args:
    task_name: {task}
    samples_jsonl: bbh/samples_{task}.jsonl
    CoT: False

bbh-{task}-3shot-CoT:
  id: bbh-{task}.val.ab-v2
  metrics: [accuracy]
bbh-{task}.val.ab-v2:
  class: evals.elsuite.dataset_specific.bbh:BBH
  args:
    task_name: {task}
    samples_jsonl: bbh/samples_{task}.jsonl
    CoT: True

"""

    eval_yaml += cur
with open(os.path.join(registry_path, "evals", "bbh.yaml"), "w") as f:
    f.write(eval_yaml)

evals/build_data/build_bbh.py
GPT-Fathom/GPT-Fathom
import json

from datasets import load_dataset

name = [
    "computer_network",
    "operating_system",
    "computer_architecture",
    "college_programming",
    "college_physics",
    "college_chemistry",
    "advanced_mathematics",
    "probability_and_statistics",
    "discrete_mathematics",
    "electrical_engineer",
    "metrology_engineer",
    "high_school_mathematics",
    "high_school_physics",
    "high_school_chemistry",
    "high_school_biology",
    "middle_school_mathematics",
    "middle_school_biology",
    "middle_school_physics",
    "middle_school_chemistry",
    "veterinary_medicine",
    "college_economics",
    "business_administration",
    "marxism",
    "mao_zedong_thought",
    "education_science",
    "teacher_qualification",
    "high_school_politics",
    "high_school_geography",
    "middle_school_politics",
    "middle_school_geography",
    "modern_chinese_history",
    "ideological_and_moral_cultivation",
    "logic",
    "law",
    "chinese_language_and_literature",
    "art_studies",
    "professional_tour_guide",
    "legal_professional",
    "high_school_chinese",
    "high_school_history",
    "middle_school_history",
    "civil_servant",
    "sports_science",
    "plant_protection",
    "basic_medicine",
    "clinical_medicine",
    "urban_and_rural_planner",
    "accountant",
    "fire_engineer",
    "environmental_impact_assessment_engineer",
    "tax_accountant",
    "physician",
]
import os

registry_path = os.path.join("..", "registry")


eval_yaml = ""
for item in name:
    file_name = item
    id_name = item.replace("_", "-")
    instruct_name = item.replace("_", " ")
    # print(file_name, id_name)
    cur = f"""
ceval-{id_name}-0shot:
  id: ceval-{id_name}.val.ab-v1
  metrics: [accuracy]
ceval-{id_name}.val.ab-v1:
  class: evals.elsuite.dataset_specific.ceval:CEVAL
  args:
    dataset: hf://ceval/ceval-exam?name={file_name}&split=val
    few_shot: hf://ceval/ceval-exam?name={file_name}&split=dev
    instructions: 以下是中国关于{file_name}考试的单项选择题，请选出其中的正确答案。
    no_MC_prompt: True
    num_few_shot: 0

ceval-{id_name}-5shot:
  id: ceval-{id_name}.val.ab-v2
  metrics: [accuracy]
ceval-{id_name}.val.ab-v2:
  class: evals.elsuite.dataset_specific.ceval:CEVAL
  args:
    dataset: hf://ceval/ceval-exam?name={file_name}&split=val
    few_shot: hf://ceval/ceval-exam?name={file_name}&split=dev
    instructions: 以下是中国关于{file_name}考试的单项选择题，请选出其中的正确答案。
    num_few_shot: 5
    no_MC_prompt: True

ceval-{id_name}-CoT:
  id: ceval-{id_name}.val.ab-v3
  metrics: [accuracy]
ceval-{id_name}.val.ab-v3:
  class: evals.elsuite.dataset_specific.ceval:CEVAL
  args:
    dataset: hf://ceval/ceval-exam?name={file_name}&split=val
    few_shot: hf://ceval/ceval-exam?name={file_name}&split=dev
    instructions: 以下是中国关于{file_name}考试的单项选择题，请选出其中的正确答案。
    no_MC_prompt: True
    CoT: True
    num_few_shot: 5
    Chinese: True
"""
    eval_yaml += cur
with open(os.path.join(registry_path, "evals", "ceval.yaml"), "w") as f:
    f.write(eval_yaml)

evals/build_data/build_ceval.py
GPT-Fathom/GPT-Fathom
import json
import os

from datasets import load_dataset

# Build the test samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "drop"), exist_ok=True)

file = load_dataset("drop", split="validation")
data = []
for row in file:
    pair = {
        "passage": row["passage"],
        "question": row["question"],
        "ideal": row["answers_spans"]["spans"],
    }
    data.append(pair)

file_path = os.path.join(registry_path, "data/drop/samples.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

# Build the few shot samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "drop"), exist_ok=True)

file = load_dataset("drop", split="train")
data = []
for row in file:
    pair = {
        "passage": row["passage"],
        "question": row["question"],
        "ideal": row["answers_spans"]["spans"],
    }
    data.append(pair)

data = data[:3]
file_path = os.path.join(registry_path, "data/drop/fewshot.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

evals/build_data/build_drop.py
GPT-Fathom/GPT-Fathom
"""Build the relevant file for lambada dataset"""
import csv
import json
import os

from datasets import load_dataset

# Build the test samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "gsm8k"), exist_ok=True)
samples = load_dataset("rookshanks/gsm8k", split="test")

data = []
for row in samples:
    pair = {"question": row["question"], "answer": row["answer"]}
    data.append(pair)

file_path = os.path.join(registry_path, "data/gsm8k/samples.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

# Build the few-shot samples

file = open("./gsm8k/prompt_original.txt", "r").read()
file = file.split("Question: ")
file = file[1:]
data = []
for i in range(8):
    text = file[i].split("Let's think")
    q, a = text[0], "Let's think" + text[1]
    pair = {"question": q, "answer": a}
    data.append(pair)

file_path = os.path.join(registry_path, "data/gsm8k/fewshot.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")


# # build gsm8k yaml

eval_yaml = ""
cur = f"""
gsm8k-8shotCoT:
  id: gsm8k.val.ab-v1
  metrics: [accuracy]
gsm8k.val.ab-v1:
  class: evals.elsuite.math_problem:MATH_PROBLEM
  args:
    samples_jsonl: gsm8k/samples.jsonl
    few_shot_jsonl: gsm8k/fewshot.jsonl
    num_few_shot: 8

gsm8k-5shotCoT:
  id: gsm8k.val.ab-v1
  metrics: [accuracy]
gsm8k.val.ab-v1:
  class: evals.elsuite.math_problem:MATH_PROBLEM
  args:
    samples_jsonl: gsm8k/samples.jsonl
    few_shot_jsonl: gsm8k/fewshot.jsonl
    num_few_shot: 5
"""
eval_yaml += cur
with open(os.path.join(registry_path, "evals", "gsm8k.yaml"), "w") as f:
    f.write(eval_yaml)

evals/build_data/build_gsm8k.py
GPT-Fathom/GPT-Fathom
"""Build the relevant file for hellaswag dataset"""
import json
import os

from datasets import load_dataset


def create_chat_prompt(text):
    return [
        {
            "role": "system",
            "content": "Complete the description with an appropriate ending, please answer with the letter of the correct answer\n",
        },
        {"role": "user", "content": text},
    ]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


# Build the test samples
# samples = load_dataset("hellaswag",  split="validation")
# registry_path = os.path.join("..", "registry")
# os.makedirs(os.path.join(registry_path, "data", "hellaswag"), exist_ok=True)

# data = []
# samples = [(item["ctx"], item["endings"], item["label"]) for item in samples]

# for item in samples[15:]:
#     words = item.split()
#     context = " ".join(words[:-1])
#     target = words[-1]
#     content = create_chat_prompt(context)
#     pair = {"input": content, "ideal": target}
#     data.append(pair)
# file_path = os.path.join(registry_path, "data/hellaswag/samples.jsonl")
# print(file_path)
# with open(file_path, 'w') as f:
#     for entry in data:
#         json.dump(entry, f)
#         f.write('\n')


# # Build the few shot samples

# import os
# registry_path = os.path.join("..", "registry")
# os.makedirs(os.path.join(registry_path, "data", "hellaswag"), exist_ok=True)

# data = []
# for item in samples[:15]:
#     words = item.split()
#     context = " ".join(words[:-1])
#     target = words[-1]
#     content = create_fewshot_prompt(context, target)
#     pair = {"sample": content}
#     data.append(pair)
# file_path = os.path.join(registry_path, "data/hellaswag/fewshot.jsonl")
# print(file_path)
# with open(file_path, 'w') as f:
#     for entry in data:
#         json.dump(entry, f)
#         f.write('\n')

evals/build_data/build_hellaswag.py
GPT-Fathom/GPT-Fathom
"""Build the relevant file for lambada dataset"""
import json
import os

from datasets import load_dataset


def create_chat_prompt(text):
    return [
        {
            "role": "system",
            "content": "Please answer with the word which is most likely to follow:",
        },
        {"role": "user", "content": text},
    ]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


# Build the test samples
samples = load_dataset("EleutherAI/lambada_openai", "en", split="test")

registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "lambada"), exist_ok=True)

data = []
samples = [item["text"] for item in samples]

for item in samples[15:]:
    words = item.split()
    context = " ".join(words[:-1])
    target = words[-1]
    content = create_chat_prompt(context)
    pair = {"input": content, "ideal": target}
    data.append(pair)
file_path = os.path.join(registry_path, "data/lambada/samples.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")


# Build the few shot samples

import os

registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "lambada"), exist_ok=True)

data = []
for item in samples[:15]:
    words = item.split()
    context = " ".join(words[:-1])
    target = words[-1]
    content = create_fewshot_prompt(context, target)
    pair = {"sample": content}
    data.append(pair)
file_path = os.path.join(registry_path, "data/lambada/fewshot.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

evals/build_data/build_lambada.py
GPT-Fathom/GPT-Fathom
"""Build the relevant file for math dataset"""
import csv
import json
import os

from datasets import load_dataset
from math_prompt.exemplars import MATH_EXEMPLARS

from evals.utils.math_util import (
    clean_numbers,
    is_equiv,
    last_boxed_only,
    last_boxed_only_string,
    remove_boxed,
)

# Build the test samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "math"), exist_ok=True)
samples = load_dataset("competition_math", split="test")

data = []
for row in samples:
    output_str = last_boxed_only_string(row["solution"])
    output = remove_boxed(output_str)
    pair = {"question": row["problem"], "answer": output}
    data.append(pair)

file_path = os.path.join(registry_path, "data/math/samples.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

# Build the few-shot samples

data = []
for i in range(1, 5):
    q, a = MATH_EXEMPLARS[str(i)]["q"], MATH_EXEMPLARS[str(i)]["a"]
    pair = {"question": q, "answer": a}
    data.append(pair)

file_path = os.path.join(registry_path, "data/math/fewshot.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

evals/build_data/build_math.py
GPT-Fathom/GPT-Fathom
"""Build the relevant file for lambada dataset"""
import csv
import json
import os

from datasets import load_dataset
from mgsm_prompt.exemplars import MGSM_EXEMPLARS


def create_chat_prompt(text):
    return [
        {
            "role": "system",
            "content": "Follow the given examples and answer the question.",
        },
        {"role": "user", "content": text},
    ]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


language = [
    "bn",
    "de",
    "es",
    "fr",
    "ja",
    "ru",
    "sw",
    "te",
    "th",
    "zh",
]  # Test the ten language except English
# print(MGSM_EXEMPLARS)


# Build the test samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "mgsm"), exist_ok=True)

for la in language:

    data = []
    with open("./mgsm_prompt/mgsm_" + la + ".tsv", "r") as file:
        reader = csv.reader(file, delimiter="\t")
        for row in reader:
            pair = {"question": row[0], "answer": row[1]}
            data.append(pair)

    file_path = os.path.join(registry_path, "data/mgsm/samples_" + la + ".jsonl")
    print(file_path)
    with open(file_path, "w") as f:
        for entry in data:
            json.dump(entry, f)
            f.write("\n")

# Build the few-shot samples

for la in language:
    data = []
    for i in range(1, 1 + 8):
        pair = {
            "question": MGSM_EXEMPLARS[la][str(i)]["q"],
            "answer": MGSM_EXEMPLARS[la][str(i)]["a"],
        }
        data.append(pair)

    file_path = os.path.join(registry_path, "data/mgsm/fewshot_" + la + ".jsonl")
    print(file_path)
    with open(file_path, "w") as f:
        for entry in data:
            json.dump(entry, f)
            f.write("\n")


# build mgsm yaml

eval_yaml = ""
for la in language:
    cur = f"""
mgsm-{la}-8shotCoT:
  id: mgsm-{la}.val.ab-v1
  metrics: [accuracy]
mgsm-{la}.val.ab-v1:
  class: evals.elsuite.basic.math_problem:MATH_PROBLEM
  args:
    samples_jsonl: mgsm/samples_{la}.jsonl
    few_shot_jsonl: mgsm/fewshot_{la}.jsonl
    num_few_shot: 6

"""
    eval_yaml += cur
with open(os.path.join(registry_path, "evals", "mgsm.yaml"), "w") as f:
    f.write(eval_yaml)

evals/build_data/build_mgsm.py
GPT-Fathom/GPT-Fathom
import json

from datasets import load_dataset

name = [
    "abstract_algebra",
    "anatomy",
    "astronomy",
    "business_ethics",
    "clinical_knowledge",
    "college_biology",
    "college_chemistry",
    "college_computer_science",
    "college_mathematics",
    "college_medicine",
    "college_physics",
    "computer_security",
    "conceptual_physics",
    "econometrics",
    "electrical_engineering",
    "elementary_mathematics",
    "formal_logic",
    "global_facts",
    "high_school_biology",
    "high_school_chemistry",
    "high_school_computer_science",
    "high_school_european_history",
    "high_school_geography",
    "high_school_government_and_politics",
    "high_school_macroeconomics",
    "high_school_mathematics",
    "high_school_microeconomics",
    "high_school_physics",
    "high_school_psychology",
    "high_school_statistics",
    "high_school_us_history",
    "high_school_world_history",
    "human_aging",
    "human_sexuality",
    "international_law",
    "jurisprudence",
    "logical_fallacies",
    "machine_learning",
    "management",
    "marketing",
    "medical_genetics",
    "miscellaneous",
    "moral_disputes",
    "moral_scenarios",
    "nutrition",
    "philosophy",
    "prehistory",
    "professional_accounting",
    "professional_law",
    "professional_medicine",
    "professional_psychology",
    "public_relations",
    "security_studies",
    "sociology",
    "us_foreign_policy",
    "virology",
    "world_religions",
]

import os

registry_path = os.path.join("..", "registry")


eval_yaml = ""
for item in name:
    file_name = item
    id_name = item.replace("_", "-")
    instruct_name = item.replace("_", " ")
    # print(file_name, id_name)
    cur = f"""
mmlu-{id_name}-3shot:
  id: mmlu-{id_name}.val.ab-v1
  metrics: [accuracy]
mmlu-{id_name}.val.ab-v1:
  class: evals.elsuite.basic.multiple_choice:MultipleChoice
  args:
    dataset: hf://hendrycks_test?name={file_name}&split=test
    few_shot: hf://hendrycks_test?name={file_name}&split=dev
    instructions: The following are multiple choice questions (with answers) about {instruct_name}.
    no_MC_prompt: True
    num_few_shot: 3

mmlu-{id_name}-5shot:
  id: mmlu-{id_name}.val.ab-v2
  metrics: [accuracy]
mmlu-{id_name}.val.ab-v2:
  class: evals.elsuite.basic.multiple_choice:MultipleChoice
  args:
    dataset: hf://hendrycks_test?name={file_name}&split=test
    few_shot: hf://hendrycks_test?name={file_name}&split=dev
    instructions: The following are multiple choice questions (with answers) about {instruct_name}.
    num_few_shot: 5
    no_MC_prompt: True

mmlu-{id_name}-CoT:
  id: mmlu-{id_name}.val.ab-v3
  metrics: [accuracy]
mmlu-{id_name}.val.ab-v3:
  class: evals.elsuite.dataset_specific.mmlu_CoT:MC_CoT
  args:
    dataset: hf://hendrycks_test?name={file_name}&split=test
    task_name: {file_name}
    no_MC_prompt: True
    CoT: True
"""
    eval_yaml += cur
with open(os.path.join(registry_path, "evals", "mmlu.yaml"), "w") as f:
    f.write(eval_yaml)

evals/build_data/build_mmlu.py
GPT-Fathom/GPT-Fathom
"""Build the relevant file for natural question dataset"""
import json
import os

from datasets import load_dataset


def create_chat_prompt(text):
    return [
        {"role": "system", "content": "Please answer the question:"},
        {"role": "user", "content": text},
    ]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


# Build the test samples
samples = load_dataset("lucadiliello/naturalquestionsshortqa", split="validation")

registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "NaturalQuestion"), exist_ok=True)

data = []

for item in samples:
    context = item["question"]
    content = create_chat_prompt(context)
    pair = {"input": content, "ideal": item["answers"]}
    data.append(pair)
file_path = os.path.join(registry_path, "data/NaturalQuestion/samples.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")


# Build the few shot samples

registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "NaturalQuestion"), exist_ok=True)
samples = load_dataset("lucadiliello/naturalquestionsshortqa", split="train")

data = []
samples = [(item["question"], item["answers"]) for item in samples]
for item in samples[:64]:
    context = item[0]
    content = create_fewshot_prompt(context, item[1][0])
    pair = {"sample": content}
    data.append(pair)
file_path = os.path.join(registry_path, "data/NaturalQuestion/fewshot.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

evals/build_data/build_natural_questions.py
GPT-Fathom/GPT-Fathom
import json
import os

from datasets import load_dataset

# Build the test samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "race"), exist_ok=True)

file = load_dataset("race", "high", split="test")
data = []
for row in file:
    pair = {
        "passage": row["article"],
        "question": row["question"],
        "options": row["options"],
        "ideal": ord(row["answer"]) - ord("A"),
    }
    data.append(pair)

file_path = os.path.join(registry_path, "data/race/samples_h.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

# Build the few shot samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "race"), exist_ok=True)

file = load_dataset("race", "high", split="test")
data = []
for row in file:
    pair = {
        "passage": row["article"],
        "question": row["question"],
        "options": row["options"],
        "ideal": ord(row["answer"]) - ord("A"),
    }
    data.append(pair)

data = data[:10]
file_path = os.path.join(registry_path, "data/race/fewshot_h.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")


# Build the test samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "race"), exist_ok=True)

file = load_dataset("race", "middle", split="test")
data = []
for row in file:
    pair = {
        "passage": row["article"],
        "question": row["question"],
        "options": row["options"],
        "ideal": ord(row["answer"]) - ord("A"),
    }
    data.append(pair)

file_path = os.path.join(registry_path, "data/race/samples_m.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

# Build the few shot samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "race"), exist_ok=True)

file = load_dataset("race", "middle", split="test")
data = []
for row in file:
    pair = {
        "passage": row["article"],
        "question": row["question"],
        "options": row["options"],
        "ideal": ord(row["answer"]) - ord("A"),
    }
    data.append(pair)

data = data[:10]
file_path = os.path.join(registry_path, "data/race/fewshot_m.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

evals/build_data/build_race.py
GPT-Fathom/GPT-Fathom
import json
import os

from datasets import load_dataset


def create_chat_prompt(text):
    return [{"role": "user", "content": text}]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


# Build the test samples for realtoxic dataset
samples = load_dataset("allenai/real-toxicity-prompts", split="train")
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "realtoxic"), exist_ok=True)

data = []

for item in samples:
    context = item["prompt"]["text"]
    content = create_chat_prompt(context)
    pair = {"input": content, "ideal": "placeholder"}
    data.append(pair)

file_path = os.path.join(registry_path, "data/realtoxic/samples.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

evals/build_data/build_realtoxic.py
GPT-Fathom/GPT-Fathom
import json
import os

from datasets import load_dataset


def create_chat_prompt(text):
    return [
        {
            "role": "system",
            "content": "Follow the given examples and answer the question.",
        },
        {"role": "user", "content": text},
    ]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


# Build the test samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "triviaqa"), exist_ok=True)

file = load_dataset("trivia_qa", "rc.nocontext", split="validation")
data = []
for row in file:
    content = create_chat_prompt(row["question"])
    pair = {
        "input": content,
        "ideal": row["answer"]["aliases"] + row["answer"]["normalized_aliases"],
    }
    data.append(pair)

file_path = os.path.join(registry_path, "data/triviaqa/samples.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

# Build the few shot samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "triviaqa"), exist_ok=True)

file = load_dataset("trivia_qa", "rc.nocontext", split="train")
data = []
for row in file:
    content = create_fewshot_prompt(row["question"], row["answer"]["aliases"][0])
    pair = {"sample": content}
    data.append(pair)

data = data[:64]
file_path = os.path.join(registry_path, "data/triviaqa/fewshot.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

evals/build_data/build_triviaqa.py
GPT-Fathom/GPT-Fathom
import json
import os

from datasets import load_dataset


def create_chat_prompt(text):
    return [
        {
            "role": "system",
            "content": "Please answer with the word which is most likely to follow:",
        },
        {"role": "user", "content": text},
    ]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


# Build the test samples for truthfulqa dataset
samples = load_dataset("truthful_qa", "multiple_choice", split="validation")
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "truthfulqa"), exist_ok=True)


data = []

for item in samples:
    pair = {
        "question": item["question"],
        "choices": item["mc1_targets"]["choices"],
        "answer": item["mc1_targets"]["labels"].index(1),
    }
    data.append(pair)

fewshot = data[0]
test_samples = data[1:]

file_path = os.path.join(registry_path, "data/truthfulqa/samples.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in test_samples:
        json.dump(entry, f)
        f.write("\n")

file_path = os.path.join(registry_path, "data/truthfulqa/fewshot.jsonl")
print(file_path)
with open(file_path, "w") as f:
        json.dump(fewshot, f)
        f.write("\n")

evals/build_data/build_truthfulqa.py
GPT-Fathom/GPT-Fathom
# khalidalt/tydiqa-goldp

import csv
import json
import os

from datasets import load_dataset


def create_chat_prompt(text):
    return [
        {
            "role": "system",
            "content": "Follow the given examples and answer the question.",
        },
        {"role": "user", "content": text},
    ]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


# Build the test samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "tydi"), exist_ok=True)

language = [
    "arabic",
    "bengali",
    "english",
    "finnish",
    "indonesian",
    "japanese",
    "korean",
    "russian",
    "swahili",
    "telugu",
    "thai",
]
for la in language:
    file = load_dataset("khalidalt/tydiqa-goldp", la, split="validation")
    data = []
    for row in file:
        content = create_chat_prompt(row["passage_text"] + "\n" + row["question_text"])
        pair = {"input": content, "ideal": row["answers"]["text"]}
        data.append(pair)

    file_path = os.path.join(registry_path, "data/tydiqa/samples_" + la + ".jsonl")
    print(file_path)
    with open(file_path, "w") as f:
        for entry in data:
            json.dump(entry, f)
            f.write("\n")


# Build the few-shot samples
for la in language:
    file = load_dataset("khalidalt/tydiqa-goldp", la, split="train")
    data = []
    for row in file:
        content = create_fewshot_prompt(
            row["passage_text"] + "\n" + row["question_text"], row["answers"]["text"]
        )
        pair = {"sample": content}
        data.append(pair)
        break  # we only need one-shot

    file_path = os.path.join(registry_path, "data/tydiqa/fewshot_" + la + ".jsonl")
    print(file_path)
    with open(file_path, "w") as f:
        for entry in data:
            json.dump(entry, f)
            f.write("\n")

# For no context:  ------------------------
# Build the test samples for languages
registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "tydi"), exist_ok=True)

language = [
    "arabic",
    "bengali",
    "english",
    "finnish",
    "indonesian",
    "japanese",
    "korean",
    "russian",
    "swahili",
    "telugu",
    "thai",
]
for la in language:
    file = load_dataset("khalidalt/tydiqa-goldp", la, split="validation")
    data = []
    for row in file:
        content = create_chat_prompt(row["question_text"])
        pair = {"input": content, "ideal": row["answers"]["text"]}
        data.append(pair)

    file_path = os.path.join(
        registry_path, "data/tydiqa/samples_nocontext_" + la + ".jsonl"
    )
    print(file_path)
    with open(file_path, "w") as f:
        for entry in data:
            json.dump(entry, f)
            f.write("\n")


# Build the few-shot samples
for la in language:
    file = load_dataset("khalidalt/tydiqa-goldp", la, split="train")
    data = []
    for row in file:
        content = create_fewshot_prompt(row["question_text"], row["answers"]["text"][0])
        pair = {"sample": content}
        # pair = {"input": row["question_text"], "ideal": row["answers"]["text"]}
        data.append(pair)
        break  # we only need one-shot

    file_path = os.path.join(
        registry_path, "data/tydiqa/fewshot_nocontext_" + la + ".jsonl"
    )
    print(file_path)
    with open(file_path, "w") as f:
        for entry in data:
            json.dump(entry, f)
            f.write("\n")


# build tydi yaml

eval_yaml = ""
for la in language:
    cur = f"""
tydi-{la}-1shotgold:
  id: tydi-{la}.val.ab-v1
  metrics: [f1_score]
tydi-{la}.val.ab-v1:
  class: evals.elsuite.basic.fuzzy_match:FuzzyMatch
  args:
    samples_jsonl: tydiqa/samples_{la}.jsonl
    few_shot_jsonl: tydiqa/fewshot_{la}.jsonl
    num_few_shot: 1
tydi-{la}-1shotnocontext:
  id: tydi-{la}.val.ab-v2
  metrics: [f1_score]
tydi-{la}.val.ab-v2:
  class: evals.elsuite.basic.fuzzy_match:FuzzyMatch
  args:
    samples_jsonl: tydiqa/samples_nocontext_{la}.jsonl
    few_shot_jsonl: tydiqa/fewshot_nocontext_{la}.jsonl
    num_few_shot: 1
"""
    eval_yaml += cur
with open(os.path.join(registry_path, "evals", "tydi.yaml"), "w") as f:
    f.write(eval_yaml)

evals/build_data/build_tydiqa.py
GPT-Fathom/GPT-Fathom
"""Build the relevant file for natural question dataset"""
import json
import os

from datasets import load_dataset


def create_chat_prompt(text):
    return [
        {"role": "system", "content": "Please answer the question:"},
        {"role": "user", "content": text},
    ]


def create_fewshot_prompt(question, answer):
    return [
        {"role": "system", "name": "example_user", "content": question},
        {"role": "system", "name": "example_assistant", "content": answer},
    ]


# Build the test samples
samples = load_dataset("web_questions", split="test")

registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "WebQ"), exist_ok=True)

data = []

for item in samples:
    context = item["question"]
    content = create_chat_prompt(context)
    pair = {"input": content, "ideal": item["answers"]}
    data.append(pair)
file_path = os.path.join(registry_path, "data/WebQ/samples.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")


# Build the few shot samples

registry_path = os.path.join("..", "registry")
os.makedirs(os.path.join(registry_path, "data", "WebQ"), exist_ok=True)
samples = load_dataset("web_questions", split="train")

data = []
samples = [(item["question"], item["answers"]) for item in samples]
for item in samples[:64]:
    context = item[0]
    # content = create_chat_prompt(context)
    # pair = {"input": content, "ideal": item["answers"]}
    # in few shot, only one example should be output
    content = create_fewshot_prompt(context, item[1][0])
    pair = {"sample": content}
    data.append(pair)
file_path = os.path.join(registry_path, "data/WebQ/fewshot.jsonl")
print(file_path)
with open(file_path, "w") as f:
    for entry in data:
        json.dump(entry, f)
        f.write("\n")

evals/build_data/build_web_questions.py
GPT-Fathom/GPT-Fathom
import os
from datasets import load_dataset

registry_path = os.path.join("..", "registry")

# MMLU
name = [
    "abstract_algebra",
    "anatomy",
    "astronomy",
    "business_ethics",
    "clinical_knowledge",
    "college_biology",
    "college_chemistry",
    "college_computer_science",
    "college_mathematics",
    "college_medicine",
    "college_physics",
    "computer_security",
    "conceptual_physics",
    "econometrics",
    "electrical_engineering",
    "elementary_mathematics",
    "formal_logic",
    "global_facts",
    "high_school_biology",
    "high_school_chemistry",
    "high_school_computer_science",
    "high_school_european_history",
    "high_school_geography",
    "high_school_government_and_politics",
    "high_school_macroeconomics",
    "high_school_mathematics",
    "high_school_microeconomics",
    "high_school_physics",
    "high_school_psychology",
    "high_school_statistics",
    "high_school_us_history",
    "high_school_world_history",
    "human_aging",
    "human_sexuality",
    "international_law",
    "jurisprudence",
    "logical_fallacies",
    "machine_learning",
    "management",
    "marketing",
    "medical_genetics",
    "miscellaneous",
    "moral_disputes",
    "moral_scenarios",
    "nutrition",
    "philosophy",
    "prehistory",
    "professional_accounting",
    "professional_law",
    "professional_medicine",
    "professional_psychology",
    "public_relations",
    "security_studies",
    "sociology",
    "us_foreign_policy",
    "virology",
    "world_religions",
]
for item in name:
    path = os.path.join(registry_path, "cached_hf_data", "mmlu", item, "test")
    if not os.path.exists(path):
        os.makedirs(path)
    dataset = load_dataset("hendrycks_test", name=item, split="test")
    dataset.save_to_disk(path)

    path = os.path.join(registry_path, "cached_hf_data", "mmlu", item, "fewshot")
    if not os.path.exists(path):
        os.makedirs(path)
    dataset = load_dataset("hendrycks_test", name=item, split="dev")
    dataset.save_to_disk(path)

# C-Eval
# name = [
#     "computer_network",
#     "operating_system",
#     "computer_architecture",
#     "college_programming",
#     "college_physics",
#     "college_chemistry",
#     "advanced_mathematics",
#     "probability_and_statistics",
#     "discrete_mathematics",
#     "electrical_engineer",
#     "metrology_engineer",
#     "high_school_mathematics",
#     "high_school_physics",
#     "high_school_chemistry",
#     "high_school_biology",
#     "middle_school_mathematics",
#     "middle_school_biology",
#     "middle_school_physics",
#     "middle_school_chemistry",
#     "veterinary_medicine",
#     "college_economics",
#     "business_administration",
#     "marxism",
#     "mao_zedong_thought",
#     "education_science",
#     "teacher_qualification",
#     "high_school_politics",
#     "high_school_geography",
#     "middle_school_politics",
#     "middle_school_geography",
#     "modern_chinese_history",
#     "ideological_and_moral_cultivation",
#     "logic",
#     "law",
#     "chinese_language_and_literature",
#     "art_studies",
#     "professional_tour_guide",
#     "legal_professional",
#     "high_school_chinese",
#     "high_school_history",
#     "middle_school_history",
#     "civil_servant",
#     "sports_science",
#     "plant_protection",
#     "basic_medicine",
#     "clinical_medicine",
#     "urban_and_rural_planner",
#     "accountant",
#     "fire_engineer",
#     "environmental_impact_assessment_engineer",
#     "tax_accountant",
#     "physician",
# ]
# for item in name:
#     path = os.path.join(registry_path, "cached_hf_data", "ceval", item, "test")
#     if not os.path.exists(path):
#         os.makedirs(path)
#     dataset = load_dataset("ceval/ceval-exam", name=item, split="val")
#     dataset.save_to_disk(path)
#     path = os.path.join(registry_path, "cached_hf_data", "ceval", item, "fewshot")
#     if not os.path.exists(path):
#         os.makedirs(path)
#     dataset = load_dataset("ceval/ceval-exam", name=item, split="dev")
#     dataset.save_to_disk(path)

# # Hellaswag
# path = os.path.join(registry_path, "cached_hf_data", "hellaswag", "validation")
# if not os.path.exists(path):
#     os.makedirs(path)
# dataset = load_dataset("hellaswag", name=item, split="validation")
# dataset.save_to_disk(path)

# path = os.path.join(registry_path, "cached_hf_data", "hellaswag", "fewshot")
# if not os.path.exists(path):
#     os.makedirs(path)
# dataset = load_dataset("hellaswag", name=item, split="train")
# dataset.save_to_disk(path)


# # coding
# dataset = load_dataset("openai_humaneval", split="test")
# path = os.path.join(registry_path, "cached_hf_data", "humaneval", "test")
# if not os.path.exists(path):
#     os.makedirs(path)
# dataset.save_to_disk(path)

# dataset = load_dataset("mbpp", "sanitized", split="prompt")
# path = os.path.join(registry_path, "cached_hf_data", "mbpp", "fewshot")
# if not os.path.exists(path):
#     os.makedirs(path)
# dataset.save_to_disk(path)

# dataset = load_dataset("mbpp", "sanitized", split="test")
# path = os.path.join(registry_path, "cached_hf_data", "mbpp", "test")
# if not os.path.exists(path):
#     os.makedirs(path)
# dataset.save_to_disk(path)

# # Winogrande
# dataset = load_dataset("winogrande", "winogrande_xl", split="validation")
# path = os.path.join(registry_path, "cached_hf_data", "winogrand", "test")
# if not os.path.exists(path):
#     os.makedirs(path)
# dataset.save_to_disk(path)


# dataset = load_dataset("winogrande", "winogrande_xl", split="train")
# path = os.path.join(registry_path, "cached_hf_data", "winogrand", "fewshot")
# if not os.path.exists(path):
#     os.makedirs(path)
# dataset.save_to_disk(path)

evals/build_data/cache_hf_data_for_local_eval.py
GPT-Fathom/GPT-Fathom
MATH_EXEMPLARS = {
    "1": {
        "q": "Find the domain of the expression $\frac{\sqrt{x-2}}{\sqrt{5-x}}$.",
        "a": "The expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\boxed{[2,5)}$."
        + "\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.",
    },
    "2": {
        "q": "If $\det \mathbf{A} = 2$ and $\det \mathbf{B} = 12,$ then find $\det (\mathbf{A} \mathbf{B}).$",
        "a": "We have that $\det (\mathbf{A} \mathbf{B}) = (\det \mathbf{A})(\det \mathbf{B}) = (2)(12) = \boxed{24}.$"
        + "\nFinal Answer: The final answer is $24$. I hope it is correct.",
    },
    "3": {
        "q": "Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?",
        "a": r"""If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight. 
                If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight. 
                Equating this to 480 pounds, we can solve for $n$: 
                \begin{align*} 
                30n&=480\\ 
                \Rightarrow\qquad n&=480/30=\boxed{16} 
                \end{align*} 
                Final Answer: The final answer is $16$. I hope it is correct.""",
    },
    "4": {
        "q": r"""If the system of equations
                \begin{align*}
                6x-4y&=a,\\
                6y-9x &=b.
                \end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero,
                find $\frac{a}{b},$ assuming $b$ is nonzero.
        """,
        "a": r"""If we multiply the first equation by $-\frac{3}{2}$, we obtain
                $$6y-9x=-\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have
                $$-\frac{3}{2}a=b\Rightarrow\frac{a}{b}=\boxed{-\frac{2}{3}}.$$
                Final Answer: The final answer is $-\frac{2}{3}$. I hope it is correct.
        """,
    },
}

evals/build_data/math_prompt/exemplars.py
GPT-Fathom/GPT-Fathom
"""Prompts for mgsm."""
# pylint: disable=line-too-long

EXEMPLAR_NUMBER_ANSWERS = [11, 29, 39, 9, 33, 8, 8, 5]

EXEMPLAR_EQUATION_SOLUTIONS = [
    "5 + 6 = 11.",
    "4 * 5 = 20. 9 + 20 = 29.",
    "32 + 42 = 74. 74 - 35 = 39.",
    "5 + 2 = 7. 7 + 2 = 9.",
    "58 - 23 = 35. 35 - 2 = 33.",
    "5 * 3 = 15. 23 - 15 = 8.",
    "20 - 12 = 8.",
    "3 + 2 = 5.",
]

MGSM_EXEMPLARS = {
    "bn": {
        "1": {
            "q": "\u09aa\u09cd\u09b0\u09b6\u09cd\u09a8: "
            "\u09b0\u099c\u09be\u09b0\u09c7\u09b0 5\u099f\u09bf "
            "\u099f\u09c7\u09a8\u09bf\u09b8 \u09ac\u09b2 "
            "\u0986\u099b\u09c7\u0964 \u09b8\u09c7 \u0986\u09b0\u0993 2 "
            "\u0995\u09cd\u09af\u09be\u09a8 \u099f\u09c7\u09a8\u09bf\u09b8"
            " \u09ac\u09b2 \u0995\u09bf\u09a8\u09c7\u099b\u09c7\u0964 "
            "\u09aa\u09cd\u09b0\u09a4\u09bf\u099f\u09bf "
            "\u0995\u09cd\u09af\u09be\u09a8\u09c7 3\u099f\u09bf "
            "\u0995\u09b0\u09c7 \u099f\u09c7\u09a8\u09bf\u09b8 "
            "\u09ac\u09b2 \u0986\u099b\u09c7\u0964 \u09a4\u09be\u09b0 "
            "\u0995\u09be\u099b\u09c7 \u098f\u0996\u09a8 "
            "\u0995\u09a4\u0997\u09c1\u09b2\u09bf "
            "\u099f\u09c7\u09a8\u09bf\u09b8 \u09ac\u09b2 "
            "\u0986\u099b\u09c7?",
            "a": "\u09a7\u09be\u09aa\u09c7 \u09a7\u09be\u09aa\u09c7 "
            "\u0989\u09a4\u09cd\u09a4\u09b0: "
            "\u09b0\u099c\u09be\u09b0\u09c7\u09b0 "
            "\u09aa\u09cd\u09b0\u09a5\u09ae\u09c7 5\u099f\u09bf "
            "\u09ac\u09b2 \u099b\u09bf\u09b2\u0964 2\u099f\u09bf "
            "\u0995\u09cd\u09af\u09be\u09a8\u09c7\u09b0 "
            "\u09aa\u09cd\u09b0\u09a4\u09bf\u099f\u09bf\u09a4\u09c7 "
            "3\u099f\u09c7 \u099f\u09c7\u09a8\u09bf\u09b8 \u09ac\u09b2 "
            "\u09ae\u09be\u09a8\u09c7 6\u099f\u09bf "
            "\u099f\u09c7\u09a8\u09bf\u09b8 \u09ac\u09b2\u0964 5 + 6 = "
            "11\u0964 \u0989\u09a4\u09cd\u09a4\u09b0 \u09b9\u09b2 11\u0964",
        },
        "2": {
            "q": "\u09aa\u09cd\u09b0\u09b6\u09cd\u09a8: "
            "\u09b8\u09be\u09b0\u09cd\u09ad\u09be\u09b0 "
            "\u0995\u0995\u09cd\u09b7\u09c7 \u09a8\u09df\u099f\u09bf "
            "\u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0 "
            "\u099b\u09bf\u09b2\u0964 \u09b8\u09cb\u09ae\u09ac\u09be\u09b0"
            " \u09a5\u09c7\u0995\u09c7 "
            "\u09ac\u09c3\u09b9\u09b8\u09cd\u09aa\u09a4\u09bf\u09ac\u09be\u09b0"
            " \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09bf\u09a8 "
            "\u0986\u09b0\u0993 \u09aa\u09be\u0981\u099a\u099f\u09bf "
            "\u0995\u09b0\u09c7 "
            "\u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0 "
            "\u09b8\u09cd\u09a5\u09be\u09aa\u09a8 \u0995\u09b0\u09be "
            "\u09b9\u09df\u09c7\u099b\u09bf\u09b2\u0964 "
            "\u09b8\u09be\u09b0\u09cd\u09ad\u09be\u09b0 "
            "\u0995\u0995\u09cd\u09b7\u09c7 \u098f\u0996\u09a8 "
            "\u0995\u09a4\u0997\u09c1\u09b2\u09bf "
            "\u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0 "
            "\u0986\u099b\u09c7?",
            "a": "\u09a7\u09be\u09aa\u09c7 \u09a7\u09be\u09aa\u09c7 "
            "\u0989\u09a4\u09cd\u09a4\u09b0: "
            "\u09b8\u09cb\u09ae\u09ac\u09be\u09b0 \u09a5\u09c7\u0995\u09c7"
            " "
            "\u09ac\u09c3\u09b9\u09b8\u09cd\u09aa\u09a4\u09bf\u09ac\u09be\u09b0"
            " 4\u09a6\u09bf\u09a8 \u09b9\u09df\u0964 "
            "\u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09bf\u09a8 "
            "5\u099f\u09bf \u0995\u09b0\u09c7 "
            "\u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0 "
            "\u09af\u09cb\u0997 \u0995\u09b0\u09be "
            "\u09b9\u09df\u09c7\u099b\u09c7\u0964 \u09af\u09be\u09b0 "
            "\u0985\u09b0\u09cd\u09a5 \u09ae\u09cb\u099f 4 * 5 = "
            "20\u099f\u09bf "
            "\u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0 "
            "\u09af\u09cb\u0997 \u0995\u09b0\u09be "
            "\u09b9\u09df\u09c7\u099b\u09c7\u0964 "
            "\u09b6\u09c1\u09b0\u09c1\u09a4\u09c7 9\u099f\u09bf "
            "\u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0 "
            "\u099b\u09bf\u09b2, \u09a4\u09be\u0987 \u098f\u0996\u09a8 9 +"
            " 20 = 29\u099f\u09bf "
            "\u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0 "
            "\u09b0\u09df\u09c7\u099b\u09c7\u0964 "
            "\u0989\u09a4\u09cd\u09a4\u09b0 \u09b9\u09b2 29\u0964",
        },
        "3": {
            "q": "\u09aa\u09cd\u09b0\u09b6\u09cd\u09a8: "
            "\u09b2\u09bf\u09df\u09be\u09b0 32\u099f\u09bf "
            "\u099a\u0995\u09cb\u09b2\u09c7\u099f \u099b\u09bf\u09b2 "
            "\u098f\u09ac\u0982 \u09a4\u09be\u09b0 "
            "\u09ac\u09cb\u09a8\u09c7\u09b0 \u099b\u09bf\u09b2 "
            "42\u099f\u09bf\u0964 \u09af\u09a6\u09bf "
            "\u09a4\u09be\u09b0\u09be 35\u099f\u09bf "
            "\u0996\u09c7\u09df\u09c7 \u09a5\u09be\u0995\u09c7, "
            "\u09a4\u09be\u09b9\u09b2\u09c7 \u09a4\u09be\u09a6\u09c7\u09b0"
            " \u0995\u09be\u099b\u09c7 \u09ae\u09cb\u099f "
            "\u0995\u09a4\u0997\u09c1\u09b2\u09bf "
            "\u0985\u09ac\u09bf\u09b6\u09bf\u09b7\u09cd\u099f "
            "\u0986\u099b\u09c7?",
            "a": "\u09a7\u09be\u09aa\u09c7 \u09a7\u09be\u09aa\u09c7 "
            "\u0989\u09a4\u09cd\u09a4\u09b0: "
            "\u09b2\u09bf\u09df\u09be\u09b0 32\u099f\u09bf "
            "\u099a\u0995\u09cb\u09b2\u09c7\u099f \u099b\u09bf\u09b2 "
            "\u098f\u09ac\u0982 \u09b2\u09bf\u09df\u09be\u09b0 "
            "\u09ac\u09cb\u09a8\u09c7\u09b0 \u099b\u09bf\u09b2 "
            "42\u099f\u09bf\u0964 \u09af\u09be\u09b0 "
            "\u0985\u09b0\u09cd\u09a5 \u09b6\u09c1\u09b0\u09c1\u09a4\u09c7"
            " 32 + 42 = 74\u099f\u09bf "
            "\u099a\u0995\u09cb\u09b2\u09c7\u099f \u099b\u09bf\u09b2\u0964"
            " 35\u099f\u09bf \u0996\u09be\u0993\u09df\u09be "
            "\u09b9\u09df\u09c7 \u0997\u09c7\u099b\u09c7\u0964 "
            "\u09a4\u09be\u0987 \u09a4\u09be\u09a6\u09c7\u09b0 "
            "\u0995\u09be\u099b\u09c7 \u09ae\u09cb\u099f 74 - 35 = "
            "39\u099f\u09bf \u099a\u0995\u09cb\u09b2\u099f "
            "\u0986\u099b\u09c7\u0964 \u0989\u09a4\u09cd\u09a4\u09b0 "
            "\u09b9\u09b2 39\u0964",
        },
        "4": {
            "q": "\u09aa\u09cd\u09b0\u09b6\u09cd\u09a8: "
            "\u09b6\u09a8\u09c7\u09b0 \u09aa\u09be\u0981\u099a\u099f\u09bf"
            " \u0996\u09c7\u09b2\u09a8\u09be \u0986\u099b\u09c7\u0964 "
            "\u0995\u09cd\u09b0\u09bf\u09b8\u09ae\u09be\u09b8 "
            "\u0989\u09aa\u09b2\u0995\u09cd\u09b7\u09c7 \u09b8\u09c7 "
            "\u09a4\u09be\u09b0 \u09ae\u09be\u09a4\u09be \u0993 "
            "\u09aa\u09bf\u09a4\u09be \u0989\u09ad\u09df\u09c7\u09b0 "
            "\u09a5\u09c7\u0995\u09c7 \u09a6\u09c1\u099f\u09bf "
            "\u0995\u09b0\u09c7 \u0996\u09c7\u09b2\u09a8\u09be "
            "\u09aa\u09c7\u09df\u09c7\u099b\u09c7\u0964 \u09a4\u09be\u09b0"
            " \u0995\u09be\u099b\u09c7 \u098f\u0996\u09a8 "
            "\u0995\u09a4\u0997\u09c1\u09b2\u09bf "
            "\u0996\u09c7\u09b2\u09a8\u09be \u0986\u099b\u09c7?",
            "a": "\u09a7\u09be\u09aa\u09c7 \u09a7\u09be\u09aa\u09c7 "
            "\u0989\u09a4\u09cd\u09a4\u09b0: \u09a4\u09be\u09b0 "
            "\u0995\u09be\u099b\u09c7 5\u099f\u09bf "
            "\u0996\u09c7\u09b2\u09a8\u09be \u0986\u099b\u09c7\u0964 "
            "\u09b8\u09c7 \u09a4\u09be\u09b0 "
            "\u09ae\u09be\u09a4\u09be\u09b0 \u09a5\u09c7\u0995\u09c7 "
            "2\u099f\u09bf \u0996\u09c7\u09b2\u09a8\u09be "
            "\u09aa\u09c7\u09df\u09c7\u099b\u09bf\u09b2 "
            "\u0985\u09a4\u098f\u09ac, \u098f\u09b0\u09aa\u09b0 "
            "\u09a4\u09be\u09b0 5 + 2 = 7\u099f\u09bf "
            "\u0996\u09c7\u09b2\u09a8\u09be "
            "\u09b9\u09df\u09c7\u099b\u09c7\u0964 "
            "\u09a4\u09be\u09b0\u09aa\u09b0 \u09b8\u09c7 "
            "\u09a4\u09be\u09b0 \u09aa\u09bf\u09a4\u09be\u09b0 "
            "\u09a5\u09c7\u0995\u09c7 2\u099f\u09bf "
            "\u0996\u09c7\u09b2\u09a8\u09be "
            "\u09aa\u09c7\u09df\u09c7\u099b\u09bf\u09b2, "
            "\u09a4\u09be\u0987 \u09a4\u09be\u09b0 \u09ae\u09cb\u099f 7 + "
            "2 = 9 \u099f\u09bf \u0996\u09c7\u09b2\u09a8\u09be "
            "\u09b9\u09df\u09c7\u099b\u09c7\u0964 "
            "\u0989\u09a4\u09cd\u09a4\u09b0 \u09b9\u09b2 9\u0964",
        },
        "5": {
            "q": "\u09aa\u09cd\u09b0\u09b6\u09cd\u09a8: "
            "\u09ae\u09be\u0987\u0995\u09c7\u09b2\u09c7\u09b0 "
            "58\u099f\u09bf \u0997\u09b2\u09ab \u09ac\u09b2 "
            "\u099b\u09bf\u09b2\u0964 "
            "\u09ae\u0999\u09cd\u0997\u09b2\u09ac\u09be\u09b0, "
            "\u09b8\u09c7 23\u099f\u09bf \u0997\u09b2\u09ab \u09ac\u09b2 "
            "\u09b9\u09be\u09b0\u09bf\u09df\u09c7\u099b\u09bf\u09b2\u0964 "
            "\u09ac\u09c1\u09a7\u09ac\u09be\u09b0, \u09b8\u09c7 "
            "\u0986\u09b0\u0993 2\u099f\u09bf \u09ac\u09b2 "
            "\u09b9\u09be\u09b0\u09bf\u09df\u09c7\u099b\u09bf\u09b2\u0964 "
            "\u09ac\u09c1\u09a7\u09ac\u09be\u09b0\u09c7\u09b0 "
            "\u09b6\u09c7\u09b7\u09c7 \u09a4\u09be\u09b0 "
            "\u0995\u09be\u099b\u09c7 \u0995\u09df\u099f\u09bf "
            "\u0997\u09b2\u09ab \u09ac\u09b2 \u099b\u09bf\u09b2?",
            "a": "\u09a7\u09be\u09aa\u09c7 \u09a7\u09be\u09aa\u09c7 "
            "\u0989\u09a4\u09cd\u09a4\u09b0: "
            "\u09b6\u09c1\u09b0\u09c1\u09a4\u09c7 "
            "\u09ae\u09be\u0987\u0995\u09c7\u09b2\u09c7\u09b0 "
            "\u0995\u09be\u099b\u09c7 58\u099f\u09bf \u0997\u09b2\u09ab "
            "\u09ac\u09b2 \u099b\u09bf\u09b2 \u098f\u09ac\u0982 "
            "\u09b8\u09c7 23\u099f\u09bf \u09ac\u09b2 "
            "\u09b9\u09be\u09b0\u09bf\u09df\u09c7\u099b\u09bf\u09b2, "
            "\u09a4\u09be\u0987 \u09a4\u09be\u09b0 58 - 23 = "
            "35\u099f\u09bf \u09ac\u09b2 \u0986\u099b\u09c7\u0964 "
            "\u0986\u09b0\u0993 2\u099f\u09bf \u09ac\u09b2 "
            "\u09b9\u09be\u09b0\u09be\u09a8\u09cb\u09b0 \u09aa\u09b0, "
            "\u09a4\u09be\u09b0 \u098f\u0996\u09a8 35 - 2 = 33\u099f\u09bf"
            " \u09ac\u09b2 \u0986\u099b\u09c7\u0964 "
            "\u0989\u09a4\u09cd\u09a4\u09b0 \u09b9\u09b2 33\u0964",
        },
        "6": {
            "q": "\u09aa\u09cd\u09b0\u09b6\u09cd\u09a8: "
            "\u0985\u09b2\u09bf\u09ad\u09bf\u09df\u09be\u09b0 $23 "
            "\u0986\u099b\u09c7\u0964 \u09b8\u09c7 "
            "\u09aa\u09be\u0981\u099a\u099f\u09bf "
            "\u09ac\u09cd\u09af\u09be\u0997\u09c7\u09b2 "
            "\u0995\u09bf\u09a8\u09c7\u099b\u09bf\u09b2 \u09af\u09be\u09b0"
            " \u09aa\u09cd\u09b0\u09a4\u09bf\u099f\u09bf\u09b0 "
            "\u09ae\u09c2\u09b2\u09cd\u09af $3\u0964 \u09a4\u09be\u09b0 "
            "\u0995\u09be\u099b\u09c7 \u0995\u09a4 "
            "\u0985\u09b0\u09cd\u09a5 "
            "\u0985\u09ac\u09b6\u09bf\u09b7\u09cd\u099f "
            "\u0986\u099b\u09c7?",
            "a": "\u09a7\u09be\u09aa\u09c7 \u09a7\u09be\u09aa\u09c7 "
            "\u0989\u09a4\u09cd\u09a4\u09b0: "
            "\u09aa\u09cd\u09b0\u09a4\u09bf\u099f\u09bf "
            "\u09ac\u09cd\u09af\u09be\u0997\u09c7\u09b2\u09c7\u09b0 "
            "\u09ae\u09c2\u09b2\u09cd\u09af $3 \u09b9\u09b2\u09c7 "
            "5\u099f\u09bf "
            "\u09ac\u09cd\u09af\u09be\u0997\u09c7\u09b2\u09c7\u09b0 "
            "\u09ae\u09c2\u09b2\u09cd\u09af \u09b9\u0993\u09df\u09be "
            "\u0989\u099a\u09bf\u09ce 5 *3 = 15 "
            "\u09a1\u09b2\u09be\u09b0\u0964 "
            "\u09b6\u09c1\u09b0\u09c1\u09a4\u09c7 "
            "\u0985\u09b2\u09bf\u09ad\u09bf\u09df\u09be\u09b0 "
            "\u0995\u09be\u099b\u09c7 $23 \u099b\u09bf\u09b2, "
            "\u09a4\u09be\u0987 "
            "\u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u09c7 "
            "\u09a4\u09be\u09b0 \u0995\u09be\u099b\u09c7 23 - 15 = 8 "
            "\u09a1\u09b2\u09be\u09b0 "
            "\u0985\u09ac\u09b6\u09bf\u09b7\u09cd\u099f "
            "\u0986\u099b\u09c7\u0964 \u0989\u09a4\u09cd\u09a4\u09b0 "
            "\u09b9\u09b2 8\u0964",
        },
        "7": {
            "q": "\u09aa\u09cd\u09b0\u09b6\u09cd\u09a8: "
            "\u099c\u09cd\u09af\u09be\u09b8\u09a8\u09c7\u09b0 "
            "20\u099f\u09bf \u09b2\u09b2\u09bf\u09aa\u09aa "
            "\u099b\u09bf\u09b2\u0964 \u09b8\u09c7 "
            "\u09a1\u09c7\u09a8\u09bf\u0995\u09c7 \u0995\u09bf\u099b\u09c1"
            " \u09b2\u09b2\u09bf\u09aa\u09aa "
            "\u09a6\u09bf\u09df\u09c7\u099b\u09bf\u09b2\u0964 "
            "\u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u09c7 "
            "\u099c\u09cd\u09af\u09be\u09b8\u09a8\u09c7\u09b0 "
            "\u0995\u09be\u099b\u09c7 12\u099f\u09bf "
            "\u09b2\u09b2\u09bf\u09aa\u09aa \u0986\u099b\u09c7\u0964 "
            "\u099c\u09cd\u09af\u09be\u09b8\u09a8 "
            "\u09a1\u09c7\u09a8\u09bf\u0995\u09c7 "
            "\u0995\u09a4\u0997\u09c1\u09b2\u09bf "
            "\u09b2\u09b2\u09bf\u09aa\u09aa "
            "\u09a6\u09bf\u09df\u09c7\u099b\u09bf\u09b2?",
            "a": "\u09a7\u09be\u09aa\u09c7 \u09a7\u09be\u09aa\u09c7 "
            "\u0989\u09a4\u09cd\u09a4\u09b0: "
            "\u099c\u09cd\u09af\u09be\u09b8\u09a8\u09c7\u09b0 "
            "\u0995\u09be\u099b\u09c7 \u09aa\u09cd\u09b0\u09a5\u09ae\u09c7"
            " 20\u099f\u09bf \u09b2\u09b2\u09bf\u09aa\u09aa "
            "\u099b\u09bf\u09b2, \u0995\u09bf\u09a8\u09cd\u09a4\u09c1 "
            "\u098f\u0996\u09a8 \u09a4\u09be\u09b0 "
            "\u0995\u09be\u099b\u09c7 "
            "\u09b6\u09c1\u09a7\u09c1\u09ae\u09be\u09a4\u09cd\u09b0 "
            "12\u099f\u09bf \u0986\u099b\u09c7, "
            "\u09a4\u09be\u09b9\u09b2\u09c7 \u09b8\u09c7 "
            "\u09a1\u09c7\u09a8\u09bf\u0995\u09c7 20 - 12 = 8\u099f\u09bf "
            "\u09b2\u09b2\u09bf\u09aa\u09aa "
            "\u09a6\u09bf\u09df\u09c7\u099b\u09bf\u09b2\u0964 "
            "\u0989\u09a4\u09cd\u09a4\u09b0 \u09b9\u09b2 8\u0964",
        },
        "8": {
            "q": "\u09aa\u09cd\u09b0\u09b6\u09cd\u09a8: "
            "\u09aa\u09be\u09b0\u09cd\u0995\u09bf\u0982 \u09b2\u099f\u09c7"
            " \u09af\u09a6\u09bf 3 \u099f\u09bf \u0997\u09be\u09dc\u09bf "
            "\u09a5\u09be\u0995\u09c7 \u098f\u09ac\u0982 "
            "\u0986\u09b0\u0993 2\u099f\u09bf \u0997\u09be\u09dc\u09bf "
            "\u0986\u09b8\u09c7, \u09a4\u09be\u09b9\u09b2\u09c7 "
            "\u09aa\u09be\u09b0\u09cd\u0995\u09bf\u0982 "
            "\u09b2\u099f\u099f\u09bf\u09a4\u09c7 "
            "\u0995\u09a4\u0997\u09c1\u09b2\u09bf \u0997\u09be\u09dc\u09bf"
            " \u0986\u099b\u09c7?",
            "a": "\u09a7\u09be\u09aa\u09c7 \u09a7\u09be\u09aa\u09c7 "
            "\u0989\u09a4\u09cd\u09a4\u09b0: "
            "\u09b6\u09c1\u09b0\u09c1\u09a4\u09c7 "
            "\u09b8\u09c7\u0996\u09be\u09a8\u09c7 3\u099f\u09bf "
            "\u0997\u09be\u09dc\u09bf \u099b\u09bf\u09b2, "
            "\u0986\u09b0\u0993 2\u099f\u09bf \u0997\u09be\u09dc\u09bf "
            "\u0986\u09b8\u09c7, \u09a4\u09be\u09b9\u09b2\u09c7 "
            "\u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u09c7 3 +2 = "
            "5\u099f\u09bf \u0997\u09be\u09dc\u09bf "
            "\u09a5\u09be\u0995\u09be \u0989\u099a\u09bf\u09ce\u0964 "
            "\u0989\u09a4\u09cd\u09a4\u09b0 \u09b9\u09b2 5\u0964",
        },
    },
    "de": {
        "1": {
            "q": "Frage: Roger hat 5 Tennisb\u00e4lle. Er kauft noch 2 Dosen "
            "Tennisb\u00e4lle. In jeder Dose sind 3 Tennisb\u00e4lle. Wie "
            "viele Tennisb\u00e4lle hat er jetzt?",
            "a": "Schritt-f\u00fcr-Schritt-Antwort: Roger begann mit 5 "
            "B\u00e4llen. 2 Dosen von jeweils 3 Tennisb\u00e4llen macht 6 "
            "Tennisb\u00e4lle. 5 + 6 = 11. Die Antwort ist 11.",
        },
        "2": {
            "q": "Frage: Es waren neun Computer im Serverraum. Von Montag bis "
            "Donnerstag wurden jeden Tag noch f\u00fcnf Computer "
            "installiert. Wie viele Computer sind jetzt im Serverraum?",
            "a": "Schritt-f\u00fcr-Schritt-Antwort: Von Montag bis Donnerstag "
            "sind es 4 Tage. Jeden Tag kamen 5 neue Computer hinzu. Das "
            "macht insgesamt 4 x 5 = 20 Computer, die hinzugef\u00fcgt "
            "wurden. Am Anfang waren es 9 Computer, also sind es jetzt 9 +"
            " 20 =29 Computer. Die Antwort lautet 29.",
        },
        "3": {
            "q": "Frage: Leah hat 32 Pralinen und ihre Schwester hat 42. Wenn "
            "sie 35 essen, wie viele sind dann insgesamt noch \u00fcbrig?",
            "a": "Schritt-f\u00fcr-Schritt-Antwort: Leah hat 32 Pralinen und "
            "Leahs Schwester 42. Das bedeutet, dass es urspr\u00fcnglich "
            "32 + 42 =74 Pralinen waren. 35 wurden gegessen. Also haben "
            "sie insgesamt noch 74 - 35 = 39 Pralinen \u00fcbrig. Die "
            "Antwort lautet 39.",
        },
        "4": {
            "q": "Frage: Shawn hat f\u00fcnf Spielzeuge. Zu Weihnachten hat er "
            "von seiner Mama und seinem Papa jeweils zwei Spielzeuge "
            "bekommen. Wie viele Spielzeuge hat er jetzt?",
            "a": "Schritt-f\u00fcr-Schritt-Antwort: Er hat 5 Spielzeuge. Er hat"
            " 2 von seiner Mama bekommen, sodass er nun 5 + 2 = 7 "
            "Spielzeuge hat. Dann hat er noch 2 von seinem Papa bekommen, "
            "also hat er insgesamt 7 + 2 = 9 Spielzeuge. Die Antwort "
            "lautet 9.",
        },
        "5": {
            "q": "Frage: Michael hat 58 Golfb\u00e4lle. Am Dienstag hat er 23 "
            "Golfb\u00e4lle verloren. Am Mittwoch hat er 2 weitere "
            "verloren. Wie viele Golfb\u00e4lle hat er Mittwoch am Ende "
            "des Tages?",
            "a": "Schritt-f\u00fcr-Schritt-Antwort: Michael hatte anfangs 58 "
            "Golfb\u00e4lle und hat 23 verloren, sodass er 58 - 23 = 35 "
            "hat. Nachdem er 2 weitere verloren hat, hat er jetzt 35 - 2 ="
            " 33 B\u00e4lle. Die Antwort lautet 33.",
        },
        "6": {
            "q": "Frage: Olivia hat 23 US-Dollar. Sie hat f\u00fcnf Bagels "
            "f\u00fcr 3 US-Dollar pro St\u00fcck gekauft. Wie viel Geld "
            "hat sie \u00fcbrig?",
            "a": "Schritt-f\u00fcr-Schritt-Antwort: 5 Bagels f\u00fcr 3 "
            "US-Dollar pro St\u00fcck kosten 5 x 3 = 15 Dollar. Olivia hat"
            " anfangs 23 US-Dollar, also hat sie jetzt 23 - 15 = 8 Dollar "
            "\u00fcbrig. Die Antwort lautet 8.",
        },
        "7": {
            "q": "Frage: Jason hatte 20 Lutscher. Er hat Denny einige Lutscher "
            "gegeben. Jetzt hat Jason 12 Lutscher. Wie viele Lutscher hat "
            "Jason Denny gegeben?",
            "a": "Schritt-f\u00fcr-Schritt-Antwort: Jason hat mit 20 Lutschern "
            "angefangen, aber jetzt hat er nur 12, also hat er Denny 20 - "
            "12 = 8 Lutscher abgegeben. Die Antwort lautet 8.",
        },
        "8": {
            "q": "Frage: Wenn 3 Autos auf dem Parkplatz stehen und 2 weitere "
            "Autos ankommen, wie viele Autos stehen dann auf dem "
            "Parkplatz?",
            "a": "Schritt-f\u00fcr-Schritt-Antwort: Anfangs sind 3 Autos da, 2 "
            "weitere kommen an, also sind jetzt 3 + 2 = 5 Autos da. Die "
            "Antwort lautet 5.",
        },
    },
    "en": {
        "1": {
            "q": "Question: Roger has 5 tennis balls. He buys 2 more cans of "
            "tennis balls. Each can has 3 tennis balls. How many tennis "
            "balls does he have now?",
            "a": "Step-by-Step Answer: Roger started with 5 balls. 2 cans of 3 "
            "tennis balls each is 6 tennis balls. 5 + 6 = 11. The answer "
            "is 11.",
        },
        "2": {
            "q": "Question: There were nine computers in the server room. Five "
            "more computers were installed each day, from monday to "
            "thursday. How many computers are now in the server room?",
            "a": "Step-by-Step Answer: There are 4 days from monday to "
            "thursday. 5 computers were added each day. That means in "
            "total 4 * 5 = 20 computers were added. There were 9 computers"
            " in the beginning, so now there are 9 + 20 = 29 computers. "
            "The answer is 29.",
        },
        "3": {
            "q": "Question: Leah had 32 chocolates and her sister had 42. If "
            "they ate 35, how many pieces do they have left in total?",
            "a": "Step-by-Step Answer: Leah had 32 chocolates and Leah’s sister"
            " had 42. That means there were originally 32 + 42 = 74 "
            "chocolates. 35 have been eaten. So in total they still have "
            "74 - 35 = 39 chocolates. The answer is 39.",
        },
        "4": {
            "q": "Question: Shawn has five toys. For Christmas, he got two toys"
            " each from his mom and dad. How many toys does he have now?",
            "a": "Step-by-Step Answer: He has 5 toys. He got 2 from mom, so "
            "after that he has 5 + 2 = 7 toys. Then he got 2 more from "
            "dad, so in total he has 7 + 2 = 9 toys. The answer is 9.",
        },
        "5": {
            "q": "Question: Michael had 58 golf balls. On tuesday, he lost 23 "
            "golf balls. On wednesday, he lost 2 more. How many golf balls"
            " did he have at the end of wednesday?",
            "a": "Step-by-Step Answer: Michael started with 58 golf balls and "
            "lost 23, so he has 58 - 23 = 35. After he lost 2 more, he has"
            " 35 - 2 = 33 balls now. The answer is 33.",
        },
        "6": {
            "q": "Question: Olivia has $23. She bought five bagels for $3 each."
            " How much money does she have left?",
            "a": "Step-by-Step Answer: 5 bagels for $3 each should cost 5 * 3 ="
            " 15 dollars. Olivia had $23 in the beginning, so now she has "
            "23 - 15 = 8 dollars left. The answer is 8.",
        },
        "7": {
            "q": "Question: Jason had 20 lollipops. He gave Denny some "
            "lollipops. Now Jason has 12 lollipops. How many lollipops did"
            " Jason give to Denny?",
            "a": "Step-by-Step Answer: Jason started with 20 lollipops, but now"
            " he only has 12, so he gave Denny 20 - 12 = 8 lollipops. The "
            "answer is 8.",
        },
        "8": {
            "q": "Question: If there are 3 cars in the parking lot and 2 more "
            "cars arrive, how many cars are in the parking lot?",
            "a": "Step-by-Step Answer: There are 3 cars in the beginning, 2 "
            "more arrive, so now there should be 3 + 2 = 5 cars. The "
            "answer is 5.",
        },
    },
    "es": {
        "1": {
            "q": "Pregunta: Roger tiene 5 pelotas de tenis. Compra 2 tubos "
            "m\u00e1s de pelotas de tenis. Cada tubo contiene 3 pelotas de"
            " tenis. \u00bfCu\u00e1ntas pelotas de tenis tiene ahora?",
            "a": "Respuesta paso a paso: Roger comenz\u00f3 con 5 pelotas. 2 "
            "tubos de 3 pelotas de tenis cada uno equivale a 6 pelotas de "
            "tenis. 5 + 6 = 11. La respuesta es 11.",
        },
        "2": {
            "q": "Pregunta: Hab\u00eda nueve computadoras en la sala de "
            "servidores. Se instalaron cinco computadoras cada d\u00eda, "
            "de lunes a jueves. \u00bfCu\u00e1ntas computadoras hay ahora "
            "en la sala de servidores?",
            "a": "Respuesta paso a paso: Hay 4 d\u00edas de lunes a jueves. Se "
            "agregaron 5 computadoras cada d\u00eda. Eso significa que en "
            "total se agregaron 4 * 5 = 20 computadoras. Al principio, "
            "hab\u00eda 9 computadoras, por lo que ahora hay 9 + 20 = 29 "
            "computadoras. La respuesta es 29.",
        },
        "3": {
            "q": "Pregunta: Leah ten\u00eda 32 chocolates y su hermana "
            "ten\u00eda 42. Si se comieron 35, \u00bfcu\u00e1ntos "
            "chocolates les quedan en total?",
            "a": "Respuesta paso a paso: Leah ten\u00eda 32 chocolates y la "
            "hermana de Leah ten\u00eda 42. Quiere decir que originalmente"
            " hab\u00eda 32 + 42 = 74 chocolates. Se comieron 35. Por lo "
            "tanto, en total todav\u00eda tienen 74 - 35 = 39 chocolates. "
            "La respuesta es 39.",
        },
        "4": {
            "q": "Pregunta: Shawn tiene cinco juguetes. Para Navidad, "
            "recibi\u00f3 dos juguetes de su mam\u00e1 y dos de su "
            "pap\u00e1. \u00bfCu\u00e1ntos juguetes tiene ahora?",
            "a": "Respuesta paso a paso: Tiene 5 juguetes. Recibi\u00f3 2 de la"
            " mam\u00e1, por lo que despu\u00e9s de eso tiene 5 + 2 = 7 "
            "juguetes. Luego, recibi\u00f3 2 m\u00e1s del pap\u00e1, "
            "as\u00ed que en total tiene 7 + 2 = 9 juguetes. La respuesta "
            "es 9.",
        },
        "5": {
            "q": "Pregunta: Michael ten\u00eda 58 pelotas de golf. El martes, "
            "perdi\u00f3 23 pelotas de golf. El mi\u00e9rcoles, "
            "perdi\u00f3 2 m\u00e1s. \u00bfCu\u00e1ntas pelotas de golf "
            "ten\u00eda al final del mi\u00e9rcoles?",
            "a": "Respuesta paso a paso: Michael comenz\u00f3 con 58 pelotas de"
            " golf y perdi\u00f3 23, por lo tanto tiene 58 - 23 = 35. "
            "Despu\u00e9s de perder 2 m\u00e1s, ahora tiene 35 - 2 = 33 "
            "pelotas. La respuesta es 33.",
        },
        "6": {
            "q": "Pregunta: Olivia tiene $23. Compr\u00f3 cinco roscas a $3 "
            "cada una. \u00bfCu\u00e1nto dinero le queda?",
            "a": "Respuesta paso a paso: 5 roscas a $3 cada una deben costar 5 "
            "* 3 = 15 d\u00f3lares. Olivia ten\u00eda $23 al principio, "
            "por lo tanto ahora le quedan 23 - 15 = 8 d\u00f3lares. La "
            "respuesta es 8.",
        },
        "7": {
            "q": "Pregunta: Jason ten\u00eda 20 chupetines. Le dio algunos "
            "chupetines a Denny. Ahora Jason tiene 12 chupetines. "
            "\u00bfCu\u00e1ntos chupetines le dio Jason a Denny?",
            "a": "Respuesta paso a paso: Jason comenz\u00f3 con 20 chupetines, "
            "pero ahora solo tiene 12, por lo tanto le dio a Denny 20 - 12"
            " = 8 chupetines. La respuesta es 8.",
        },
        "8": {
            "q": "Pregunta: Si hay 3 autos en el estacionamiento y llegan 2 "
            "autos m\u00e1s, \u00bfcu\u00e1ntos autos hay en el "
            "estacionamiento?",
            "a": "Respuesta paso a paso: Hay 3 autos al principio, llegan 2 "
            "m\u00e1s, por lo tanto ahora debe haber 3 + 2 = 5 autos. La "
            "respuesta es 5.",
        },
    },
    "fr": {
        "1": {
            "q": "Question : Roger a 5 balles de tennis. Il ach\u00e8te 2 "
            "autres bo\u00eetes de balles de tennis en plus. Si chaque "
            "bo\u00eete contient 3 balles de tennis, combien de balles de "
            "tennis a-t-il maintenant ?",
            "a": "R\u00e9ponse \u00e9tape par \u00e9tape : Roger a "
            "commenc\u00e9 avec 5 balles. 2 bo\u00eetes de 3 balles de "
            "tennis chacune repr\u00e9sentent 6 balles de tennis. 5 + 6 = "
            "11. La r\u00e9ponse est 11.",
        },
        "2": {
            "q": "Question : Il y avait neuf ordinateurs dans la salle des "
            "serveurs. Cinq ordinateurs suppl\u00e9mentaires ont "
            "\u00e9t\u00e9 install\u00e9s chaque jour, du lundi au jeudi. "
            "Combien d'ordinateurs y a-t-il maintenant dans la salle des "
            "serveurs ?",
            "a": "R\u00e9ponse \u00e9tape par \u00e9tape : De lundi \u00e0 "
            "jeudi, il y a 4 jours. 5 ordinateurs ont \u00e9t\u00e9 "
            "ajout\u00e9s par jour. Cela signifie qu'au total, 4 x 5 = 20 "
            "ordinateurs ont \u00e9t\u00e9 ajout\u00e9s. Comme il y avait "
            "9 ordinateurs au d\u00e9part, il y a maintenant 9 + 20 = 29 "
            "ordinateurs. La r\u00e9ponse est 29.",
        },
        "3": {
            "q": "Question : L\u00e9a avait 32 chocolats et sa s\u0153ur en "
            "avait 42. Si elles en ont mang\u00e9 35, combien de morceaux "
            "leur reste-t-il en tout ?",
            "a": "R\u00e9ponse \u00e9tape par \u00e9tape : L\u00e9a avait 32 "
            "chocolats et sa s\u0153ur en avait 42. Cela signifie qu'il y "
            "avait \u00e0 l'origine 32 + 42 = 74 chocolats. 35 chocolats "
            "ont \u00e9t\u00e9 mang\u00e9s. Il leur en reste donc au total"
            " 74 - 35 = 39 chocolats. La r\u00e9ponse est 39.",
        },
        "4": {
            "q": "Question : Shawn a cinq jouets. Pour No\u00ebl, papa et maman"
            " lui ont offert deux jouets chacun. Combien de jouets a-t-il "
            "maintenant ?",
            "a": "R\u00e9ponse \u00e9tape par \u00e9tape : Il a 5 jouets. Il en"
            " a re\u00e7u 2 de maman, ce qui fait qu'il a 5 + 2 = 7 "
            "jouets. Ensuite, il a re\u00e7u 2 autres jouets de papa. Cela"
            " fait donc un total de 7 + 2 = 9 jouets. La r\u00e9ponse est "
            "9.",
        },
        "5": {
            "q": "Question : Michael avait 58 balles de golf. Mardi, il a perdu"
            " 23 balles de golf. Mercredi, il en a perdu 2 autres. De "
            "combien de balles de golf disposait-il \u00e0 la fin du "
            "mercredi ?",
            "a": "R\u00e9ponse \u00e9tape par \u00e9tape : Michael a "
            "commenc\u00e9 avec 58 balles de golf et en a perdu 23. Il lui"
            " reste donc 58 - 23 = 35. Apr\u00e8s en avoir perdu 2 autres,"
            " il lui reste 35 - 2 = 33 balles. La r\u00e9ponse est 33.",
        },
        "6": {
            "q": "Question : Olivia a 23 $. Elle a achet\u00e9 cinq bagels "
            "\u00e0 3 $ chacun. Combien d'argent lui reste-t-il ?",
            "a": "R\u00e9ponse \u00e9tape par \u00e9tape : 5 bagels \u00e0 3 $ "
            "chacun co\u00fbtent 5 x 3 = 15 dollars. Olivia avait 23 "
            "dollars au d\u00e9part, il lui reste donc 23 - 15 = 8 "
            "dollars. La r\u00e9ponse est 8.",
        },
        "7": {
            "q": "Question : Jason avait 20 sucettes. Il en a donn\u00e9 \u00e0"
            " Denny et il ne lui en reste plus que 12 \u00e0 pr\u00e9sent."
            " Combien de sucettes Jason a-t-il donn\u00e9 \u00e0 Denny ?",
            "a": "R\u00e9ponse \u00e9tape par \u00e9tape : Au d\u00e9part, "
            "Jason avait 20 sucettes. Il n'en a plus que 12, alors il a "
            "donn\u00e9 \u00e0 Denny 20 - 12 = 8 sucettes. La r\u00e9ponse"
            " est 8.",
        },
        "8": {
            "q": "Question : S'il y a 3 voitures dans le parking et que 2 "
            "autres arrivent, combien de voitures y a-t-il maintenant dans"
            " le parking ?",
            "a": "R\u00e9ponse \u00e9tape par \u00e9tape : Au d\u00e9but, il y "
            "a 3 voitures, puis 2 autres arrivent. \u00c7a fait donc 3 + 2"
            " = 5 voitures. La r\u00e9ponse est 5.",
        },
    },
    "ja": {
        "1": {
            "q": "\u554f\u984c\uff1a\u30ed\u30b8\u30e3\u30fc\u306f5\u500b\u306e\u30c6\u30cb\u30b9\u30dc\u30fc\u30eb\u304c\u3042\u308a\u307e\u3059\u3002\u30c6\u30cb\u30b9\u30dc\u30fc\u30eb\u306e\u7f36\u30922\u3064\u8ffd\u52a0\u3067\u8cb7\u3044\u307e\u3059\u3002\u305d\u308c\u305e\u308c\u306e\u7f36\u306b\u306f3\u3064\u306e\u30c6\u30cb\u30b9\u30dc\u30fc\u30eb\u304c\u5165\u3063\u3066\u3044\u307e\u3059\u3002\u5f7c\u306f\u4eca\u3044\u304f\u3064\u306e\u30c6\u30cb\u30b9\u30dc\u30fc\u30eb\u304c\u3042\u308a\u307e\u3059\u304b\uff1f",
            "a": "\u30b9\u30c6\u30c3\u30d7\u3054\u3068\u306e\u7b54\u3048\uff1a\u30ed\u30b8\u30e3\u30fc\u306f\u6700\u521d5\u500b\u306e\u30dc\u30fc\u30eb\u304c\u3042\u308a\u307e\u3057\u305f\u3002\u30c6\u30cb\u30b9\u30dc\u30fc\u30eb3\u500b\u5165\u308a\u306e\u7f36\u304c2\u3064\u3042\u308c\u3070\u3001\u30c6\u30cb\u30b9\u30dc\u30fc\u30eb\u306f6\u500b\u3042\u308a\u307e\u3059\u30025+6=11\u3002\u7b54\u3048\u306f11\u3067\u3059\u3002",
        },
        "2": {
            "q": "\u554f\u984c\uff1a\u30b5\u30fc\u30d0\u30fc\u30eb\u30fc\u30e0\u306b\u306f9\u53f0\u306e\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u30fc\u304c\u3042\u308a\u307e\u3057\u305f\u3002\u6708\u66dc\u65e5\u304b\u3089\u6728\u66dc\u65e5\u307e\u3067\u6bce\u65e55\u53f0\u306e\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u30fc\u3092\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3057\u307e\u3057\u305f\u3002\u4eca\u30b5\u30fc\u30d0\u30fc\u30eb\u30fc\u30e0\u306b\u306f\u96e3\u984c\u306e\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u30fc\u304c\u3042\u308a\u307e\u3059\u304b\uff1f",
            "a": "\u30b9\u30c6\u30c3\u30d7\u3054\u3068\u306e\u7b54\u3048\uff1a\u6708\u66dc\u304b\u3089\u6728\u66dc\u307e\u30674\u65e5\u3042\u308a\u307e\u3059\u3002\u6bce\u65e55\u53f0\u306e\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u30fc\u304c\u8ffd\u52a0\u3055\u308c\u307e\u3059\u3002\u3064\u307e\u308a\u3001\u5168\u90e8\u30674*5=20\u53f0\u306e\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u30fc\u304c\u8ffd\u52a0\u3055\u308c\u307e\u3057\u305f\u3002\u6700\u521d\u306b9\u53f0\u306e\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u30fc\u304c\u3042\u3063\u305f\u306e\u3067\u3001\u4eca\u306f9+20=29\u53f0\u306e\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u30fc\u3068\u306a\u308a\u307e\u3059\u3002\u7b54\u3048\u306f29\u3067\u3059\u3002",
        },
        "3": {
            "q": "\u554f\u984c\uff1a\u30ea\u30a2\u306f32\u500b\u306e\u30c1\u30e7\u30b3\u30ec\u30fc\u30c8\u3092\u6301\u3063\u3066\u3044\u307e\u3057\u305f\u3001\u5f7c\u5973\u306e\u59b9\u306f42\u500b\u6301\u3063\u3066\u3044\u307e\u3057\u305f\u3002\u5f7c\u5973\u9054\u304c35\u500b\u98df\u3079\u305f\u3068\u3057\u305f\u3089\u3001\u5168\u90e8\u3067\u4f55\u500b\u6b8b\u3063\u3066\u3044\u307e\u3059\u304b\uff1f",
            "a": "\u30b9\u30c6\u30c3\u30d7\u3054\u3068\u306e\u7b54\u3048\uff1a\u30ea\u30a2\u306f32\u500b\u306e\u30c1\u30e7\u30b3\u30ec\u30fc\u30c8\u3092\u6301\u3063\u3066\u304a\u308a\u3001\u30ea\u30a2\u306e\u59b9\u306f42\u500b\u6301\u3063\u3066\u3044\u305f\u3002\u3064\u307e\u308a\u3001\u5143\u300532+42=74\u500b\u306e\u30c1\u30e7\u30b3\u30ec\u30fc\u30c8\u304c\u3042\u3063\u305f\u300235\u500b\u304c\u98df\u3079\u3089\u308c\u305f\u3002\u3060\u304b\u3089\u5168\u90e8\u3067\u3001\u5f7c\u5973\u305f\u3061\u306b\u306f74-35=39\u500b\u306e\u30c1\u30e7\u30b3\u30ec\u30fc\u30c8\u304c\u6b8b\u3063\u3066\u3044\u308b\u3002\u7b54\u3048\u306f39\u500b\u3002",
        },
        "4": {
            "q": "\u554f\u984c\uff1a\u30b7\u30e7\u30fc\u30f3\u306f5\u500b\u306e\u304a\u3082\u3061\u3083\u3092\u6301\u3063\u3066\u3044\u307e\u3059\u3002\u30af\u30ea\u30b9\u30de\u30b9\u306b\u3001\u5f7c\u306f\u7236\u3068\u6bcd\u304b\u3089\u305d\u308c\u305e\u308c2\u3064\u305a\u3064\u304a\u3082\u3061\u3083\u3092\u3082\u3089\u3044\u307e\u3057\u305f\u3002\u4eca\u5f7c\u306f\u3044\u304f\u3064\u306e\u304a\u3082\u3061\u3083\u304c\u3042\u308a\u307e\u3059\u304b\uff1f",
            "a": "\u30b9\u30c6\u30c3\u30d7\u3054\u3068\u306e\u7b54\u3048\uff1a\u5f7c\u306f5\u500b\u304a\u3082\u3061\u3083\u3092\u6301\u3063\u3066\u3044\u307e\u3059\u3002\u6bcd\u304b\u30892\u3064\u3082\u3089\u3063\u305f\u5f8c\u306f\u30015+2=7\u500b\u306e\u304a\u3082\u3061\u3083\u304c\u3042\u308a\u307e\u3059\u3002\u305d\u306e\u5f8c\u7236\u304b\u30892\u3064\u3055\u3089\u306b\u3082\u3089\u3044\u3001\u5168\u90e8\u3067\u5f7c\u306f\u30017+2=9\u500b\u306e\u304a\u3082\u3061\u3083\u304c\u3042\u308a\u307e\u3059\u3002\u7b54\u3048\u306f9\u500b\u3067\u3059\u3002",
        },
        "5": {
            "q": "\u554f\u984c\uff1a\u30de\u30a4\u30b1\u30eb\u306f58\u500b\u306e\u30b4\u30eb\u30d5\u30dc\u30fc\u30eb\u3092\u6301\u3063\u3066\u3044\u307e\u3059\u3002\u706b\u66dc\u65e5\u3001\u5f7c\u306f\n23\u500b\u306e\u30b4\u30eb\u30d5\u30dc\u30fc\u30eb\u3092\u5931\u304f\u3057\u307e\u3057\u305f\u3002\u6c34\u66dc\u65e5\u3001\u3055\u3089\u306b2\u500b\u5931\u304f\u3057\u307e\u3057\u305f\u3002\u6c34\u66dc\u65e5\u306e\u7d42\u308f\u308a\u306b\u306f\u3001\u5f7c\u306f\u4f55\u3053\u306e\u30b4\u30eb\u30d5\u30dc\u30fc\u30eb\u3092\u6301\u3063\u3066\u3044\u307e\u3057\u305f\u304b\uff1f",
            "a": "\u30b9\u30c6\u30c3\u30d7\u3054\u3068\u306e\u7b54\u3048\uff1a\u30de\u30a4\u30b1\u30eb\u306f\u6700\u521d58\u500b\u306e\u30b4\u30eb\u30d5\u30dc\u30fc\u30eb\u304c\u3042\u308a\u300123\u500b\u306a\u304f\u3057\u307e\u3057\u305f\u3001\u3060\u304b\u3089\u5f7c\u306f58-23=35\u500b\u6b8b\u3063\u3066\u3044\u307e\u3059\u3002\u5f7c\u304c\u3042\u30682\u500b\u306a\u304f\u3057\u305f\u5f8c\u3001\u5f7c\u306f\u4eca35-2=33\u500b\u6b8b\u3063\u3066\u3044\u307e\u3059\u3002\u7b54\u3048\u306f33\u500b\u3067\u3059\u3002",
        },
        "6": {
            "q": "\u554f\u984c\uff1a\u30aa\u30ea\u30d3\u30a2\u306b\u306f$23\u3042\u308a\u307e\u3059\u3002\u5f7c\u5973\u306f\u305d\u308c\u305e\u308c$3\u306e\u30d9\u30fc\u30b0\u30eb\u30925\u3064\u8cb7\u3044\u307e\u3057\u305f\u3002\u5f7c\u5973\u306b\u306f\u3044\u304f\u3089\u306e\u304a\u91d1\u304c\u6b8b\u3063\u3066\u3044\u307e\u3059\u304b\uff1f",
            "a": "\u30b9\u30c6\u30c3\u30d7\u3054\u3068\u306e\u7b54\u3048\uff1a$3\u306e\u30d9\u30fc\u30b0\u30eb\u306f5\u500b\u3067\u30015*3=15\u30c9\u30eb\u306b\u306a\u308b\u306f\u305a\u3067\u3059\u3002\u30aa\u30ea\u30d3\u30a2\u306f\u6700\u521d$23\u3042\u3063\u305f\u306e\u3067\u3001\u4eca\u5f7c\u5973\u306b\u306f23-15=8\u30c9\u30eb\u6b8b\u3063\u3066\u3044\u307e\u3059\u3002\u7b54\u3048\u306f8\u3067\u3059\u3002",
        },
        "7": {
            "q": "\u554f\u984c\uff1a\u30b8\u30a7\u30a4\u30bd\u30f3\u306f20\u500b\u306e\u98f4\u3092\u6301\u3063\u3066\u3044\u307e\u3059\u3002\u5f7c\u306f\u30c7\u30cb\u30fc\u306b\u98f4\u3092\u3044\u304f\u3064\u304b\u3042\u3052\u307e\u3057\u305f\u3002\u4eca\u3001\u30b8\u30a7\u30a4\u30bd\u30f3\u306b\u306f12\u500b\u306e\u98f4\u304c\u3042\u308a\u307e\u3059\u3002\u30b8\u30a7\u30a4\u30bd\u30f3\u306f\u30c7\u30cb\u30fc\u306b\u3044\u304f\u3064\u98f4\u3092\u3042\u3052\u307e\u3057\u305f\u304b\uff1f",
            "a": "\u30b9\u30c6\u30c3\u30d7\u3054\u3068\u306e\u7b54\u3048\uff1a\u30b8\u30a7\u30a4\u30bd\u30f3\u306b\u306f\u6700\u521d20\u500b\u306e\u98f4\u304c\u3042\u308a\u307e\u3057\u305f\u304c\u3001\u4eca\u5f7c\u306f12\u500b\u3057\u304b\u6301\u3063\u3066\u3044\u307e\u305b\u3093\u3001\u306a\u306e\u3067\u5f7c\u306f\u30c7\u30cb\u30fc\u306b\u300120-12=8\u500b\u306e\u98f4\u3092\u3042\u3052\u307e\u3057\u305f\u3002\u7b54\u3048\u306f8\u3067\u3059\u3002",
        },
        "8": {
            "q": "\u554f\u984c\uff1a\u99d0\u8eca\u5834\u306b3\u53f0\u306e\u8eca\u304c\u3042\u308a\u30012\u53f0\u306e\u8eca\u304c\u5230\u7740\u3059\u308b\u3068\u3057\u305f\u3089\u3001\u99d0\u8eca\u5834\u306b\u306f\u4f55\u53f0\u306e\u8eca\u304c\u3042\u308a\u307e\u3059\u304b\uff1f",
            "a": "\u30b9\u30c6\u30c3\u30d7\u3054\u3068\u306e\u7b54\u3048\uff1a\u6700\u521d\u306b3\u53f0\u306e\u8eca\u304c\u3042\u308a\u3001\u3055\u3089\u306b2\u53f0\u5230\u7740\u3057\u305f\u306e\u3067\u3001\u4eca\u306f3+2\u30675\u53f0\u306e\u8eca\u304c\u3042\u308b\u306f\u305a\u3067\u3059\u3002\u7b54\u3048\u306f5\u3067\u3059\u3002",
        },
    },
    "ru": {
        "1": {
            "q": "\u0417\u0430\u0434\u0430\u0447\u0430: \u0443 "
            "\u0420\u043e\u0434\u0436\u0435\u0440\u0430 5 "
            "\u0442\u0435\u043d\u043d\u0438\u0441\u043d\u044b\u0445 "
            "\u043c\u044f\u0447\u0435\u0439. \u041e\u043d "
            "\u043f\u043e\u043a\u0443\u043f\u0430\u0435\u0442 "
            "\u0435\u0449\u0435 2 \u0431\u0430\u043d\u043a\u0438 "
            "\u0442\u0435\u043d\u043d\u0438\u0441\u043d\u044b\u0445 "
            "\u043c\u044f\u0447\u0435\u0439. \u0412 "
            "\u043a\u0430\u0436\u0434\u043e\u0439 "
            "\u0431\u0430\u043d\u043a\u0435 3 "
            "\u0442\u0435\u043d\u043d\u0438\u0441\u043d\u044b\u0445 "
            "\u043c\u044f\u0447\u0430. "
            "\u0421\u043a\u043e\u043b\u044c\u043a\u043e "
            "\u0442\u0435\u043d\u043d\u0438\u0441\u043d\u044b\u0445 "
            "\u043c\u044f\u0447\u0435\u0439 \u0443 "
            "\u043d\u0435\u0433\u043e "
            "\u0441\u0435\u0439\u0447\u0430\u0441?",
            "a": "\u041f\u043e\u0448\u0430\u0433\u043e\u0432\u043e\u0435 "
            "\u0440\u0435\u0448\u0435\u043d\u0438\u0435: \u0432 "
            "\u043d\u0430\u0447\u0430\u043b\u0435 \u0443 "
            "\u0420\u043e\u0434\u0436\u0435\u0440\u0430 "
            "\u0431\u044b\u043b\u043e 5 \u043c\u044f\u0447\u0435\u0439. 2 "
            "\u0431\u0430\u043d\u043a\u0438 \u043f\u043e 3 "
            "\u0442\u0435\u043d\u043d\u0438\u0441\u043d\u044b\u0445 "
            "\u043c\u044f\u0447\u0430 \u043a\u0430\u0436\u0434\u0430\u044f"
            " \u2014 \u044d\u0442\u043e 6 "
            "\u0442\u0435\u043d\u043d\u0438\u0441\u043d\u044b\u0445 "
            "\u043c\u044f\u0447\u0435\u0439. 5 + 6 = 11. "
            "\u041e\u0442\u0432\u0435\u0442 \u2014 11.",
        },
        "2": {
            "q": "\u0417\u0430\u0434\u0430\u0447\u0430: \u0432 "
            "\u0441\u0435\u0440\u0432\u0435\u0440\u043d\u043e\u0439 "
            "\u0431\u044b\u043b\u043e \u0434\u0435\u0432\u044f\u0442\u044c"
            " "
            "\u043a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u043e\u0432."
            " \u041a\u0430\u0436\u0434\u044b\u0439 "
            "\u0434\u0435\u043d\u044c \u0441 "
            "\u043f\u043e\u043d\u0435\u0434\u0435\u043b\u044c\u043d\u0438\u043a\u0430"
            " \u043f\u043e \u0447\u0435\u0442\u0432\u0435\u0440\u0433 "
            "\u0443\u0441\u0442\u0430\u043d\u0430\u0432\u043b\u0438\u0432\u0430\u043b\u043e\u0441\u044c"
            " \u0435\u0449\u0435 \u043f\u043e \u043f\u044f\u0442\u044c "
            "\u043a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u043e\u0432."
            " \u0421\u043a\u043e\u043b\u044c\u043a\u043e "
            "\u043a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u043e\u0432"
            " \u0441\u0435\u0439\u0447\u0430\u0441 \u0432 "
            "\u0441\u0435\u0440\u0432\u0435\u0440\u043d\u043e\u0439?"
            "\u043a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u043e\u0432.",
            "a": "\u041f\u043e\u0448\u0430\u0433\u043e\u0432\u043e\u0435 "
            "\u0440\u0435\u0448\u0435\u043d\u0438\u0435: \u0441 "
            "\u043f\u043e\u043d\u0435\u0434\u0435\u043b\u044c\u043d\u0438\u043a\u0430"
            " \u043f\u043e \u0447\u0435\u0442\u0432\u0435\u0440\u0433 "
            "\u043f\u0440\u043e\u0448\u043b\u043e 4 \u0434\u043d\u044f. "
            "\u041a\u0430\u0436\u0434\u044b\u0439 \u0434\u0435\u043d\u044c"
            " \u0434\u043e\u0431\u0430\u0432\u043b\u044f\u043b\u0438 "
            "\u043f\u043e 5 "
            "\u043a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u043e\u0432."
            " \u0422\u043e \u0435\u0441\u0442\u044c "
            "\u0432\u0441\u0435\u0433\u043e "
            "\u0434\u043e\u0431\u0430\u0432\u0438\u043b\u0438 4 * 5 = 20 "
            "\u043a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u043e\u0432."
            " \u0412\u043d\u0430\u0447\u0430\u043b\u0435 "
            "\u0431\u044b\u043b\u043e 9 "
            "\u043a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u043e\u0432,"
            " \u0437\u043d\u0430\u0447\u0438\u0442, "
            "\u0442\u0435\u043f\u0435\u0440\u044c "
            "\u0441\u0442\u0430\u043b\u043e 9 + 20 = 29 "
            "\u043a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u043e\u0432."
            " \u041e\u0442\u0432\u0435\u0442 \u2014 29.",
        },
        "3": {
            "q": "\u0417\u0430\u0434\u0430\u0447\u0430: \u0443 "
            "\u041b\u0438\u0438 \u0431\u044b\u043b\u043e 32 "
            "\u0448\u043e\u043a\u043e\u043b\u0430\u0434\u043d\u044b\u0445 "
            "\u043a\u043e\u043d\u0444\u0435\u0442\u044b, \u0430 \u0443 "
            "\u0435\u0435 \u0441\u0435\u0441\u0442\u0440\u044b \u2014 42. "
            "\u0415\u0441\u043b\u0438 35 \u043e\u043d\u0438 "
            "\u0441\u044a\u0435\u043b\u0438, "
            "\u0441\u043a\u043e\u043b\u044c\u043a\u043e "
            "\u0432\u0441\u0435\u0433\u043e \u0448\u0442\u0443\u043a "
            "\u0443 \u043d\u0438\u0445 "
            "\u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c?",
            "a": "\u041f\u043e\u0448\u0430\u0433\u043e\u0432\u043e\u0435 "
            "\u0440\u0435\u0448\u0435\u043d\u0438\u0435: \u0443 "
            "\u041b\u0438\u0438 \u0431\u044b\u043b\u043e 32 "
            "\u0448\u043e\u043a\u043e\u043b\u0430\u0434\u043d\u044b\u0445 "
            "\u043a\u043e\u043d\u0444\u0435\u0442\u044b, \u0430 \u0443 "
            "\u0441\u0435\u0441\u0442\u0440\u044b \u041b\u0438\u0438 "
            "\u2014 42. \u0417\u043d\u0430\u0447\u0438\u0442, "
            "\u043f\u0435\u0440\u0432\u043e\u043d\u0430\u0447\u0430\u043b\u044c\u043d\u043e"
            " \u0431\u044b\u043b\u043e 32 + 42 = 74 "
            "\u0448\u043e\u043a\u043e\u043b\u0430\u0434\u043d\u044b\u0445 "
            "\u043a\u043e\u043d\u0444\u0435\u0442\u044b. 35 "
            "\u0441\u044a\u0435\u043b\u0438. "
            "\u0417\u043d\u0430\u0447\u0438\u0442, "
            "\u0432\u0441\u0435\u0433\u043e "
            "\u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c 74 - 35 = 39"
            " \u0448\u043e\u043a\u043e\u043b\u0430\u0434\u043d\u044b\u0445"
            " \u043a\u043e\u043d\u0444\u0435\u0442. "
            "\u041e\u0442\u0432\u0435\u0442 \u2014 39.",
        },
        "4": {
            "q": "\u0417\u0430\u0434\u0430\u0447\u0430: \u0443 "
            "\u0428\u043e\u043d\u0430 \u0431\u044b\u043b\u043e "
            "\u043f\u044f\u0442\u044c "
            "\u0438\u0433\u0440\u0443\u0448\u0435\u043a. \u041d\u0430 "
            "\u0420\u043e\u0436\u0434\u0435\u0441\u0442\u0432\u043e "
            "\u043e\u043d \u043f\u043e\u043b\u0443\u0447\u0438\u043b "
            "\u043f\u043e \u0434\u0432\u0435 "
            "\u0438\u0433\u0440\u0443\u0448\u043a\u0438 \u043e\u0442 "
            "\u043c\u0430\u043c\u044b \u0438 \u043e\u0442 "
            "\u043f\u0430\u043f\u044b. "
            "\u0421\u043a\u043e\u043b\u044c\u043a\u043e \u0443 "
            "\u043d\u0435\u0433\u043e \u0441\u0435\u0439\u0447\u0430\u0441"
            " \u0438\u0433\u0440\u0443\u0448\u0435\u043a?",
            "a": "\u041f\u043e\u0448\u0430\u0433\u043e\u0432\u043e\u0435 "
            "\u0440\u0435\u0448\u0435\u043d\u0438\u0435: \u0443 "
            "\u043d\u0435\u0433\u043e \u0431\u044b\u043b\u043e 5 "
            "\u0438\u0433\u0440\u0443\u0448\u0435\u043a. \u041e\u043d "
            "\u043f\u043e\u043b\u0443\u0447\u0438\u043b 2 \u043e\u0442 "
            "\u043c\u0430\u043c\u044b, \u0438 "
            "\u043f\u043e\u0441\u043b\u0435 \u044d\u0442\u043e\u0433\u043e"
            " \u0443 \u043d\u0435\u0433\u043e "
            "\u0441\u0442\u0430\u043b\u043e 5 + 2 = 7 "
            "\u0438\u0433\u0440\u0443\u0448\u0435\u043a. "
            "\u041f\u043e\u0442\u043e\u043c \u043e\u043d "
            "\u043f\u043e\u043b\u0443\u0447\u0438\u043b \u0435\u0449\u0435"
            " 2 \u043e\u0442 \u043f\u0430\u043f\u044b, \u0442\u0430\u043a "
            "\u0447\u0442\u043e \u0432 \u043e\u0431\u0449\u0435\u0439 "
            "\u0441\u043b\u043e\u0436\u043d\u043e\u0441\u0442\u0438 \u0443"
            " \u043d\u0435\u0433\u043e \u0441\u0442\u0430\u043b\u043e 7 + "
            "2 = 9 \u0438\u0433\u0440\u0443\u0448\u0435\u043a. "
            "\u041e\u0442\u0432\u0435\u0442 \u2014 9.",
        },
        "5": {
            "q": "\u0417\u0430\u0434\u0430\u0447\u0430: \u0443 "
            "\u041c\u0430\u0439\u043a\u043b\u0430 \u0431\u044b\u043b\u043e"
            " 58 \u043c\u044f\u0447\u0435\u0439 \u0434\u043b\u044f "
            "\u0433\u043e\u043b\u044c\u0444\u0430. \u0412\u043e "
            "\u0432\u0442\u043e\u0440\u043d\u0438\u043a \u043e\u043d "
            "\u043f\u043e\u0442\u0435\u0440\u044f\u043b 23 "
            "\u043c\u044f\u0447\u0430 \u0434\u043b\u044f "
            "\u0433\u043e\u043b\u044c\u0444\u0430. \u0412 "
            "\u0441\u0440\u0435\u0434\u0443 \u043e\u043d "
            "\u043f\u043e\u0442\u0435\u0440\u044f\u043b \u0435\u0449\u0435"
            " 2. \u0421\u043a\u043e\u043b\u044c\u043a\u043e "
            "\u043c\u044f\u0447\u0435\u0439 \u0434\u043b\u044f "
            "\u0433\u043e\u043b\u044c\u0444\u0430 "
            "\u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c \u0443 "
            "\u043d\u0435\u0433\u043e \u043a "
            "\u043a\u043e\u043d\u0446\u0443 "
            "\u0441\u0440\u0435\u0434\u044b?",
            "a": "\u041f\u043e\u0448\u0430\u0433\u043e\u0432\u043e\u0435 "
            "\u0440\u0435\u0448\u0435\u043d\u0438\u0435: "
            "\u0432\u043d\u0430\u0447\u0430\u043b\u0435 \u0443 "
            "\u041c\u0430\u0439\u043a\u043b\u0430 \u0431\u044b\u043b\u043e"
            " 58 \u043c\u044f\u0447\u0435\u0439 \u0434\u043b\u044f "
            "\u0433\u043e\u043b\u044c\u0444\u0430, 23 \u043e\u043d "
            "\u043f\u043e\u0442\u0435\u0440\u044f\u043b, \u0438 \u0443 "
            "\u043d\u0435\u0433\u043e "
            "\u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c 58 - 23 = "
            "35. \u041f\u043e\u0441\u043b\u0435 "
            "\u044d\u0442\u043e\u0433\u043e \u043e\u043d "
            "\u043f\u043e\u0442\u0435\u0440\u044f\u043b \u0435\u0449\u0435"
            " 2, \u0438 \u0442\u0435\u043f\u0435\u0440\u044c \u0443 "
            "\u043d\u0435\u0433\u043e "
            "\u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c 35 - 2 = 33 "
            "\u043c\u044f\u0447\u0430. \u041e\u0442\u0432\u0435\u0442 "
            "\u2014 33.",
        },
        "6": {
            "q": "\u0417\u0430\u0434\u0430\u0447\u0430: \u0443 "
            "\u041e\u043b\u0438\u0432\u0438\u0438 \u0431\u044b\u043b\u043e"
            " 23 $. \u041e\u043d\u0430 "
            "\u043a\u0443\u043f\u0438\u043b\u0430 \u043f\u044f\u0442\u044c"
            " \u0440\u043e\u0433\u0430\u043b\u0438\u043a\u043e\u0432 "
            "\u043f\u043e 3 $. \u0421\u043a\u043e\u043b\u044c\u043a\u043e "
            "\u0434\u0435\u043d\u0435\u0433 \u0443 \u043d\u0435\u0435 "
            "\u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c?",
            "a": "\u041f\u043e\u0448\u0430\u0433\u043e\u0432\u043e\u0435 "
            "\u0440\u0435\u0448\u0435\u043d\u0438\u0435: 5 "
            "\u0440\u043e\u0433\u0430\u043b\u0438\u043a\u043e\u0432 "
            "\u043f\u043e 3 $ \u0441\u0442\u043e\u044f\u0442 5 * 3 = 15 "
            "\u0434\u043e\u043b\u043b\u0430\u0440\u043e\u0432. "
            "\u0412\u043d\u0430\u0447\u0430\u043b\u0435 \u0443 "
            "\u041e\u043b\u0438\u0432\u0438\u0438 \u0431\u044b\u043b\u043e"
            " 23 $, \u0437\u043d\u0430\u0447\u0438\u0442, "
            "\u0442\u0435\u043f\u0435\u0440\u044c \u0443 "
            "\u043d\u0435\u0435 "
            "\u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c 23 - 15 = 8 "
            "$. \u041e\u0442\u0432\u0435\u0442 \u2014 8.",
        },
        "7": {
            "q": "\u0417\u0430\u0434\u0430\u0447\u0430: \u0443 "
            "\u0414\u0436\u0435\u0439\u0441\u043e\u043d\u0430 "
            "\u0431\u044b\u043b\u043e 20 "
            "\u043b\u0435\u0434\u0435\u043d\u0446\u043e\u0432. "
            "\u041e\u043d \u0434\u0430\u043b "
            "\u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u043e "
            "\u043b\u0435\u0434\u0435\u043d\u0446\u043e\u0432 "
            "\u0414\u044d\u043d\u043d\u0438. "
            "\u0422\u0435\u043f\u0435\u0440\u044c \u0443 "
            "\u0414\u0436\u0435\u0439\u0441\u043e\u043d\u0430 "
            "\u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c 12 "
            "\u043b\u0435\u0434\u0435\u043d\u0446\u043e\u0432. "
            "\u0421\u043a\u043e\u043b\u044c\u043a\u043e "
            "\u043b\u0435\u0434\u0435\u043d\u0446\u043e\u0432 "
            "\u0414\u0436\u0435\u0439\u0441\u043e\u043d \u0434\u0430\u043b"
            " \u0414\u044d\u043d\u043d\u0438?",
            "a": "\u041f\u043e\u0448\u0430\u0433\u043e\u0432\u043e\u0435 "
            "\u0440\u0435\u0448\u0435\u043d\u0438\u0435: "
            "\u0432\u043d\u0430\u0447\u0430\u043b\u0435 \u0443 "
            "\u0414\u0436\u0435\u0439\u0441\u043e\u043d\u0430 "
            "\u0431\u044b\u043b\u043e 20 "
            "\u043b\u0435\u0434\u0435\u043d\u0446\u043e\u0432, \u0430 "
            "\u0442\u0435\u043f\u0435\u0440\u044c "
            "\u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c "
            "\u0442\u043e\u043b\u044c\u043a\u043e 12, "
            "\u0437\u043d\u0430\u0447\u0438\u0442, \u043e\u043d "
            "\u0434\u0430\u043b \u0414\u044d\u043d\u043d\u0438 20 - 12 = 8"
            " \u043b\u0435\u0434\u0435\u043d\u0446\u043e\u0432. "
            "\u041e\u0442\u0432\u0435\u0442 \u2014 8.",
        },
        "8": {
            "q": "\u0417\u0430\u0434\u0430\u0447\u0430: "
            "\u0435\u0441\u043b\u0438 \u043d\u0430 "
            "\u043f\u0430\u0440\u043a\u043e\u0432\u043a\u0435 3 "
            "\u0430\u0432\u0442\u043e\u043c\u043e\u0431\u0438\u043b\u044f "
            "\u0438 "
            "\u043f\u043e\u0434\u044a\u0435\u0437\u0436\u0430\u044e\u0442 "
            "\u0435\u0449\u0435 2, "
            "\u0441\u043a\u043e\u043b\u044c\u043a\u043e "
            "\u0430\u0432\u0442\u043e\u043c\u043e\u0431\u0438\u043b\u0435\u0439"
            " \u043d\u0430 "
            "\u043f\u0430\u0440\u043a\u043e\u0432\u043a\u0435?",
            "a": "\u041f\u043e\u0448\u0430\u0433\u043e\u0432\u043e\u0435 "
            "\u0440\u0435\u0448\u0435\u043d\u0438\u0435: "
            "\u0432\u043d\u0430\u0447\u0430\u043b\u0435 "
            "\u0435\u0441\u0442\u044c 3 "
            "\u0430\u0432\u0442\u043e\u043c\u043e\u0431\u0438\u043b\u044f,"
            " \u043f\u043e\u0434\u044a\u0435\u0437\u0436\u0430\u044e\u0442"
            " \u0435\u0449\u0435 2, \u0437\u043d\u0430\u0447\u0438\u0442, "
            "\u0442\u0435\u043f\u0435\u0440\u044c \u0442\u0430\u043c "
            "\u0434\u043e\u043b\u0436\u043d\u043e \u0431\u044b\u0442\u044c"
            " 3 + 2 = 5 "
            "\u0430\u0432\u0442\u043e\u043c\u043e\u0431\u0438\u043b\u0435\u0439."
            " \u041e\u0442\u0432\u0435\u0442 \u2014 5.",
        },
    },
    "sw": {
        "1": {
            "q": "Swali: Roger ana mipira 5 ya tenisi. Ananunua mikebe 2 zaidi "
            "ya mipira ya tenisi. Kila mkebe una mipira 3 ya tenisi. Ana "
            "mipira mingapi ya tenisi kwa sasa?",
            "a": "Jibu la Hatua kwa Hatua: Roger alianza kwa mipira 5. Mikebe 2"
            " yenye mipira 3 ya tenisi kila oja ni mipira 6 ya tenisi. 5 +"
            " 6 = 11. Jibu ni 11.",
        },
        "2": {
            "q": "Swali: Kuna kompyuta tisa katika chumba cha seva. Kompyuta "
            "tano zaidi zilisakinishwa kila siku, kuanzia Jumatatu hadi "
            "Alhamisi. Kuna kompyuta ngapi wkenye chumba cha seva kufikia "
            "sasa?",
            "a": "Jibu la Hatua kwa Hatua: Kuna siku 4 kuanzia Jumatatu mpaka "
            "Alhamisi. Kompyuta 5 ziliongezwa kila siku. Hiyo inamaanisha "
            "kuwa kwa jumla kompyuta 4 * 5 = 20 ziliongezwa. Kulikuwa na "
            "kompyuta 9 mwanzoni, hivyo sasa kuna kompyuta 9 + 20 = 29. "
            "Jibu ni 29.",
        },
        "3": {
            "q": "Swali: Leah alikuwa na chokoleti 32 na dadake alikuwa na 42. "
            "Iwapo walikula 35, wamesalia na chokoleti ngapi kwa jumla?",
            "a": "Jibu la Hatua kwa Hatua: Lea alikuwa na chokoleti 32 na "
            "dadake Leah alikuwa na 42. Hiyo inamaanisha kuwa awali "
            "kulikuwa na chokoleti 32 + 42 + 74. 35 zimeliwa. Hivyo kwa "
            "jumla bado kuna chokoleti 74 - 35 = 39. Jibu ni 39.",
        },
        "4": {
            "q": "Swali: Shawn ana wanasesere watano. Siku ya Krismasi, alipata"
            " wanasesere wawili kutoka kwa mamake na babake kila mmoja. "
            "Sasa ana wanasesere wangapi kwa sasa?",
            "a": "Jibu la Hatua kwa Hatua: Ana wanasesere 5. Alipata 2 kutoka "
            "kwa mama, hivyo baada ya hapo alikuwa na wanasesere 5 + 2 = "
            "7. Kisha alipata 2 zaidi kutoka kwa baba, hivyo kwa jumla "
            "alikuwa na wanasesere 7 + 2 = 9. Jibu ni 9.",
        },
        "5": {
            "q": "Swali: Michael alikuwa na mipira 58 ya gofu. Jumanne, "
            "alipoteza mipira 23 ya gofu. Jumatano, alipoteza mipira 2 "
            "zaidi. Alikuwa na mipira mingapi ya gofu kufikia mwishoni mwa"
            " Jumatano?",
            "a": "Jibu la Hatua kwa Hatua: Michael alianza na mipira 58 ya gofu"
            " na akapoteza 23, hivyo ana 58 - 23 = 35. Baadaye alipoteza 2"
            " zaidi, sasa ana mipira 35 - 2 = 33. Jibu ni 33.",
        },
        "6": {
            "q": "Swali: Olivia ana $23. Alinunua bageli tano kwa $3 kila moja."
            " Amesalia na pesa ngapi?",
            "a": "Jibu la Hatua kwa Hatua: Bageli 5 za $3 kila moja zinafaa "
            "kugharimu 5 * 3 = dola 15. Olivia alikuwa na $23 mwanzoni, "
            "sasa amesalia na dola 23 - 15 = 8. Jibu ni 8.",
        },
        "7": {
            "q": "Swali: Jason alikuwa na pipi 20. Alimpa Denny pipi nyingine, "
            "Sasa Jason amesalia na pipi 12. Jason alimpa Denny pipi "
            "ngapi?",
            "a": "Jibu la Hatua kwa Hatua: Jason alianza na pipi 20, lakini "
            "sasa ana 12, hivyo alimpa Denny pipi 20 - 12 = 8. Jibu ni 8.",
        },
        "8": {
            "q": "Swali: Ikiwa kuna magari 3 katika eneo la maegesho na magari "
            "2 zaidi yameongezeka, kuna magari mangapi kwa jumla katika "
            "eneo la maegesho?",
            "a": "Jibu la Hatua kwa Hatua: Mwanzoni kuna magari 3, 2 zaidi "
            "yameongezeka hivyo kutakuwa na magari 3 + 2 = 5. Jibu ni 5.",
        },
    },
    "te": {
        "1": {
            "q": "\u0c2a\u0c4d\u0c30\u0c36\u0c4d\u0c28: "
            "\u0c30\u0c4b\u0c1c\u0c30\u0c4d \u0c35\u0c26\u0c4d\u0c26 5 "
            "\u0c1f\u0c46\u0c28\u0c4d\u0c28\u0c3f\u0c38\u0c4d "
            "\u0c2c\u0c02\u0c24\u0c41\u0c32\u0c41 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c05\u0c24\u0c21\u0c41 \u0c2e\u0c30\u0c4b 2 "
            "\u0c15\u0c4d\u0c2f\u0c3e\u0c28\u0c4d\u200c\u0c32 "
            "\u0c1f\u0c46\u0c28\u0c4d\u0c28\u0c3f\u0c38\u0c4d "
            "\u0c2c\u0c02\u0c24\u0c41\u0c32\u0c41 "
            "\u0c15\u0c4a\u0c28\u0c41\u0c17\u0c4b\u0c32\u0c41 "
            "\u0c1a\u0c47\u0c36\u0c3e\u0c21\u0c41. "
            "\u0c2a\u0c4d\u0c30\u0c24\u0c3f "
            "\u0c15\u0c4d\u0c2f\u0c3e\u0c28\u0c4d\u200c\u0c32\u0c4b 3 "
            "\u0c1f\u0c46\u0c28\u0c4d\u0c28\u0c3f\u0c38\u0c4d "
            "\u0c2c\u0c02\u0c24\u0c41\u0c32\u0c41\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f."
            " \u0c07\u0c2a\u0c4d\u0c2a\u0c41\u0c21\u0c41 "
            "\u0c05\u0c24\u0c21\u0c3f \u0c35\u0c26\u0c4d\u0c26 "
            "\u0c0e\u0c28\u0c4d\u0c28\u0c3f "
            "\u0c1f\u0c46\u0c28\u0c4d\u0c28\u0c3f\u0c38\u0c4d "
            "\u0c2c\u0c02\u0c24\u0c41\u0c32\u0c41 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f?",
            "a": "\u0c26\u0c36\u0c32\u0c35\u0c3e\u0c30\u0c40\u0c17\u0c3e "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02: "
            "\u0c30\u0c4b\u0c1c\u0c30\u0c4d 5 "
            "\u0c2c\u0c02\u0c24\u0c41\u0c32\u0c24\u0c4b "
            "\u0c2a\u0c4d\u0c30\u0c3e\u0c30\u0c02\u0c2d\u0c3f\u0c02\u0c1a\u0c3e\u0c21\u0c41."
            " 2 "
            "\u0c15\u0c4d\u0c2f\u0c3e\u0c28\u0c4d\u200c\u0c32\u0c4d\u0c32\u0c4b"
            " \u0c12\u0c15\u0c4d\u0c15\u0c26\u0c3e\u0c28\u0c3f\u0c32\u0c4b"
            " 3 \u0c1f\u0c46\u0c28\u0c4d\u0c28\u0c3f\u0c38\u0c4d "
            "\u0c2c\u0c02\u0c24\u0c41\u0c32\u0c41 \u0c05\u0c02\u0c1f\u0c47"
            " 6 \u0c1f\u0c46\u0c28\u0c4d\u0c28\u0c3f\u0c38\u0c4d "
            "\u0c2c\u0c02\u0c24\u0c41\u0c32\u0c41 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. 5+6=11. "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02 11.",
        },
        "2": {
            "q": "\u0c2a\u0c4d\u0c30\u0c36\u0c4d\u0c28: "
            "\u0c38\u0c30\u0c4d\u0c35\u0c30\u0c4d "
            "\u0c30\u0c42\u0c2e\u0c4d\u200c\u0c32\u0c4b "
            "\u0c24\u0c4a\u0c2e\u0c4d\u0c2e\u0c3f\u0c26\u0c3f "
            "\u0c15\u0c02\u0c2a\u0c4d\u0c2f\u0c42\u0c1f\u0c30\u0c4d\u200c\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c38\u0c4b\u0c2e\u0c35\u0c3e\u0c30\u0c02 "
            "\u0c28\u0c41\u0c02\u0c1a\u0c3f "
            "\u0c17\u0c41\u0c30\u0c41\u0c35\u0c3e\u0c30\u0c02 "
            "\u0c35\u0c30\u0c15\u0c41 "
            "\u0c2a\u0c4d\u0c30\u0c24\u0c3f\u0c30\u0c4b\u0c1c\u0c42 "
            "\u0c2e\u0c30\u0c4b \u0c10\u0c26\u0c41 "
            "\u0c15\u0c02\u0c2a\u0c4d\u0c2f\u0c42\u0c1f\u0c30\u0c4d\u200c\u0c32\u0c41"
            " \u0c07\u0c28\u0c4d\u200c\u0c38\u0c4d\u0c1f\u0c3e\u0c32\u0c4d"
            " "
            "\u0c1a\u0c47\u0c2f\u0c2c\u0c21\u0c4d\u0c21\u0c3e\u0c2f\u0c3f."
            " \u0c38\u0c30\u0c4d\u0c35\u0c30\u0c4d "
            "\u0c30\u0c42\u0c2e\u0c4d\u200c\u0c32\u0c4b "
            "\u0c07\u0c2a\u0c4d\u0c2a\u0c41\u0c21\u0c41 "
            "\u0c0e\u0c28\u0c4d\u0c28\u0c3f "
            "\u0c15\u0c02\u0c2a\u0c4d\u0c2f\u0c42\u0c1f\u0c30\u0c4d\u200c\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f?"
            "\u0c15\u0c02\u0c2a\u0c4d\u0c2f\u0c42\u0c1f\u0c30\u0c4d\u200c\u0c32\u0c41"
            "\u0c15\u0c02\u0c2a\u0c4d\u0c2f\u0c42\u0c1f\u0c30\u0c4d\u200c\u0c32\u0c41",
            "a": "\u0c26\u0c36\u0c32\u0c35\u0c3e\u0c30\u0c40\u0c17\u0c3e "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02: "
            "\u0c38\u0c4b\u0c2e\u0c35\u0c3e\u0c30\u0c02 "
            "\u0c28\u0c41\u0c02\u0c1a\u0c3f "
            "\u0c17\u0c41\u0c30\u0c41\u0c35\u0c3e\u0c30\u0c02 "
            "\u0c35\u0c30\u0c15\u0c41 4 "
            "\u0c30\u0c4b\u0c1c\u0c41\u0c32\u0c41\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f."
            " \u0c2a\u0c4d\u0c30\u0c24\u0c3f\u0c30\u0c4b\u0c1c\u0c42 5 "
            "\u0c15\u0c02\u0c2a\u0c4d\u0c2f\u0c42\u0c1f\u0c30\u0c4d\u200c\u0c32\u0c41"
            " "
            "\u0c1c\u0c4b\u0c21\u0c3f\u0c02\u0c1a\u0c2c\u0c21\u0c4d\u0c21\u0c3e\u0c2f\u0c3f."
            " \u0c05\u0c02\u0c1f\u0c47 "
            "\u0c2e\u0c4a\u0c24\u0c4d\u0c24\u0c02 4*5=20 "
            "\u0c15\u0c02\u0c2a\u0c4d\u0c2f\u0c42\u0c1f\u0c30\u0c4d\u200c\u0c32\u0c41"
            " "
            "\u0c1c\u0c4b\u0c21\u0c3f\u0c02\u0c1a\u0c2c\u0c21\u0c4d\u0c21\u0c3e\u0c2f\u0c3f."
            " \u0c2a\u0c4d\u0c30\u0c3e\u0c30\u0c02\u0c2d\u0c02\u0c32\u0c4b"
            " 9 "
            "\u0c15\u0c02\u0c2a\u0c4d\u0c2f\u0c42\u0c1f\u0c30\u0c4d\u200c\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f, "
            "\u0c05\u0c02\u0c26\u0c41\u0c35\u0c32\u0c4d\u0c32 "
            "\u0c07\u0c2a\u0c4d\u0c2a\u0c41\u0c21\u0c41 \u0c05\u0c35\u0c3f"
            " 9+20=29 "
            "\u0c15\u0c02\u0c2a\u0c4d\u0c2f\u0c42\u0c1f\u0c30\u0c4d\u200c\u0c32\u0c41"
            " \u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02 29.",
        },
        "3": {
            "q": "\u0c2a\u0c4d\u0c30\u0c36\u0c4d\u0c28: "
            "\u0c32\u0c3f\u0c2f\u0c3e \u0c35\u0c26\u0c4d\u0c26 32 "
            "\u0c1a\u0c3e\u0c15\u0c4d\u0c32\u0c46\u0c1f\u0c4d\u200c\u0c32\u0c41"
            " \u0c2e\u0c30\u0c3f\u0c2f\u0c41 \u0c06\u0c2e\u0c46 "
            "\u0c38\u0c4b\u0c26\u0c30\u0c3f \u0c35\u0c26\u0c4d\u0c26 42 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c35\u0c3e\u0c30\u0c41 35 "
            "\u0c24\u0c3f\u0c28\u0c4d\u0c28\u0c1f\u0c4d\u0c32\u0c2f\u0c3f\u0c24\u0c47,"
            " \u0c2e\u0c4a\u0c24\u0c4d\u0c24\u0c02 \u0c2e\u0c40\u0c26 "
            "\u0c35\u0c3e\u0c30\u0c3f \u0c35\u0c26\u0c4d\u0c26 "
            "\u0c0e\u0c28\u0c4d\u0c28\u0c3f "
            "\u0c2a\u0c40\u0c38\u0c4d\u200c\u0c32\u0c41 "
            "\u0c2e\u0c3f\u0c17\u0c3f\u0c32\u0c3f "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f?",
            "a": "\u0c26\u0c36\u0c32\u0c35\u0c3e\u0c30\u0c40\u0c17\u0c3e "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02: "
            "\u0c32\u0c3f\u0c2f\u0c3e \u0c35\u0c26\u0c4d\u0c26 32 "
            "\u0c1a\u0c3e\u0c15\u0c4d\u0c32\u0c46\u0c1f\u0c4d\u200c\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f, "
            "\u0c32\u0c3f\u0c2f\u0c3e \u0c38\u0c4b\u0c26\u0c30\u0c3f "
            "\u0c35\u0c26\u0c4d\u0c26 42 "
            "\u0c1a\u0c3e\u0c15\u0c4d\u0c32\u0c46\u0c1f\u0c4d\u200c\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c05\u0c02\u0c1f\u0c47 \u0c35\u0c3e\u0c30\u0c3f "
            "\u0c35\u0c26\u0c4d\u0c26 "
            "\u0c35\u0c3e\u0c38\u0c4d\u0c24\u0c35\u0c02\u0c17\u0c3e 32+42="
            " 74 "
            "\u0c1a\u0c3e\u0c15\u0c4d\u0c32\u0c46\u0c1f\u0c4d\u200c\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. 35 "
            "\u0c1a\u0c3e\u0c15\u0c4d\u0c32\u0c46\u0c1f\u0c4d\u200c\u0c32\u0c28\u0c41"
            " \u0c24\u0c3f\u0c28\u0c4d\u0c28\u0c3e\u0c30\u0c41. "
            "\u0c05\u0c02\u0c26\u0c41\u0c35\u0c32\u0c4d\u0c32 "
            "\u0c35\u0c3e\u0c30\u0c3f \u0c35\u0c26\u0c4d\u0c26 "
            "\u0c2e\u0c4a\u0c24\u0c4d\u0c24\u0c02 74-35= 39 "
            "\u0c1a\u0c3e\u0c15\u0c4d\u0c32\u0c46\u0c1f\u0c4d\u200c\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02 39."
            "\u0c1a\u0c3e\u0c15\u0c4d\u0c32\u0c46\u0c1f\u0c4d\u200c\u0c32\u0c41"
            "\u0c1a\u0c3e\u0c15\u0c4d\u0c32\u0c46\u0c1f\u0c4d\u200c\u0c32\u0c41"
            "\u0c1a\u0c3e\u0c15\u0c4d\u0c32\u0c46\u0c1f\u0c4d\u200c\u0c32\u0c41",
        },
        "4": {
            "q": "\u0c2a\u0c4d\u0c30\u0c36\u0c4d\u0c28: "
            "\u0c37\u0c3e\u0c28\u0c4d \u0c35\u0c26\u0c4d\u0c26 "
            "\u0c10\u0c26\u0c41 \u0c2c\u0c4a\u0c2e\u0c4d\u0c2e\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c05\u0c24\u0c21\u0c41 "
            "\u0c15\u0c4d\u0c30\u0c3f\u0c38\u0c4d\u0c1f\u0c2e\u0c38\u0c4d "
            "\u0c15\u0c4a\u0c30\u0c15\u0c41 \u0c24\u0c28 "
            "\u0c05\u0c2e\u0c4d\u0c2e \u0c2e\u0c30\u0c3f\u0c2f\u0c41 "
            "\u0c28\u0c3e\u0c28\u0c4d\u0c28\u0c32 "
            "\u0c28\u0c41\u0c02\u0c1a\u0c3f "
            "\u0c30\u0c46\u0c02\u0c21\u0c47\u0c38\u0c3f "
            "\u0c2c\u0c4a\u0c2e\u0c4d\u0c2e\u0c32 "
            "\u0c1a\u0c4a\u0c2a\u0c4d\u0c2a\u0c41\u0c28 "
            "\u0c2a\u0c4a\u0c02\u0c26\u0c3e\u0c21\u0c41. "
            "\u0c07\u0c2a\u0c4d\u0c2a\u0c41\u0c21\u0c41 "
            "\u0c05\u0c24\u0c21\u0c3f \u0c35\u0c26\u0c4d\u0c26 "
            "\u0c0e\u0c28\u0c4d\u0c28\u0c3f "
            "\u0c2c\u0c4a\u0c2e\u0c4d\u0c2e\u0c32\u0c41 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f?",
            "a": "\u0c26\u0c36\u0c32\u0c35\u0c3e\u0c30\u0c40\u0c17\u0c3e "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02: "
            "\u0c05\u0c24\u0c21\u0c3f \u0c35\u0c26\u0c4d\u0c26 5 "
            "\u0c2c\u0c4a\u0c2e\u0c4d\u0c2e\u0c32\u0c41 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c05\u0c24\u0c21\u0c41 \u0c05\u0c2e\u0c4d\u0c2e "
            "\u0c26\u0c17\u0c4d\u0c17\u0c30 \u0c28\u0c41\u0c02\u0c1a\u0c3f"
            " 2 \u0c2a\u0c4a\u0c02\u0c26\u0c3e\u0c21\u0c41, "
            "\u0c15\u0c3e\u0c2c\u0c1f\u0c4d\u0c1f\u0c3f \u0c06 "
            "\u0c24\u0c30\u0c41\u0c35\u0c3e\u0c24 \u0c05\u0c24\u0c21\u0c3f"
            " \u0c35\u0c26\u0c4d\u0c26 5+2=7 "
            "\u0c2c\u0c4a\u0c2e\u0c4d\u0c2e\u0c32\u0c41\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f."
            " \u0c24\u0c30\u0c41\u0c35\u0c3e\u0c24 "
            "\u0c24\u0c02\u0c21\u0c4d\u0c30\u0c3f \u0c35\u0c26\u0c4d\u0c26"
            " \u0c28\u0c41\u0c02\u0c1a\u0c3f \u0c05\u0c24\u0c21\u0c41 "
            "\u0c2e\u0c30\u0c4b 2 "
            "\u0c2a\u0c4a\u0c02\u0c26\u0c3e\u0c21\u0c41. "
            "\u0c05\u0c02\u0c26\u0c41\u0c35\u0c32\u0c4d\u0c32 "
            "\u0c2e\u0c4a\u0c24\u0c4d\u0c24\u0c02 \u0c2e\u0c40\u0c26 "
            "\u0c05\u0c24\u0c21\u0c3f \u0c35\u0c26\u0c4d\u0c26 7+2=9 "
            "\u0c2c\u0c4a\u0c2e\u0c4d\u0c2e\u0c32\u0c41\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f."
            " \u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02 9."
            "\u0c2c\u0c4a\u0c2e\u0c4d\u0c2e\u0c32\u0c41\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f.",
        },
        "5": {
            "q": "\u0c2a\u0c4d\u0c30\u0c36\u0c4d\u0c28: "
            "\u0c2e\u0c48\u0c15\u0c46\u0c32\u0c4d \u0c35\u0c26\u0c4d\u0c26"
            " 58 \u0c17\u0c4b\u0c32\u0c4d\u0c2b\u0c4d "
            "\u0c2c\u0c3e\u0c32\u0c4d\u0c38\u0c4d "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c2e\u0c02\u0c17\u0c33\u0c35\u0c3e\u0c30\u0c02 "
            "\u0c28\u0c3e\u0c21\u0c41, \u0c05\u0c24\u0c21\u0c41 23 "
            "\u0c17\u0c4b\u0c32\u0c4d\u0c2b\u0c4d "
            "\u0c2c\u0c3e\u0c32\u0c4d\u0c38\u0c4d\u200c\u0c28\u0c3f "
            "\u0c2a\u0c4a\u0c17\u0c4a\u0c1f\u0c4d\u0c1f\u0c41\u0c15\u0c41\u0c28\u0c4d\u0c28\u0c3e\u0c21\u0c41."
            " \u0c2c\u0c41\u0c27\u0c35\u0c3e\u0c30\u0c02 "
            "\u0c28\u0c3e\u0c21\u0c41, \u0c05\u0c24\u0c21\u0c41 "
            "\u0c2e\u0c30\u0c4b 2 "
            "\u0c2a\u0c4b\u0c17\u0c4a\u0c1f\u0c4d\u0c1f\u0c41\u0c15\u0c41\u0c28\u0c4d\u0c28\u0c3e\u0c21\u0c41."
            " \u0c2c\u0c41\u0c27\u0c35\u0c3e\u0c30\u0c02 "
            "\u0c1a\u0c3f\u0c35\u0c30\u0c3f\u0c15\u0c3f "
            "\u0c05\u0c24\u0c21\u0c3f \u0c35\u0c26\u0c4d\u0c26 "
            "\u0c0e\u0c28\u0c4d\u0c28\u0c3f "
            "\u0c17\u0c4b\u0c32\u0c4d\u0c2b\u0c4d "
            "\u0c2c\u0c3e\u0c32\u0c4d\u200c\u0c32\u0c41 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f?",
            "a": "\u0c26\u0c36\u0c32\u0c35\u0c3e\u0c30\u0c40\u0c17\u0c3e "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02: "
            "\u0c2e\u0c48\u0c15\u0c46\u0c32\u0c4d 58 "
            "\u0c17\u0c4b\u0c32\u0c4d\u0c2b\u0c4d "
            "\u0c2c\u0c3e\u0c32\u0c4d\u0c38\u0c4d\u200c\u0c24\u0c4b "
            "\u0c2a\u0c4d\u0c30\u0c3e\u0c30\u0c02\u0c2d\u0c3f\u0c02\u0c1a\u0c3e\u0c21\u0c41"
            " \u0c2e\u0c30\u0c3f\u0c2f\u0c41 23 "
            "\u0c2a\u0c4b\u0c17\u0c4a\u0c1f\u0c4d\u0c1f\u0c41\u0c15\u0c41\u0c28\u0c4d\u0c28\u0c3e\u0c21\u0c41,"
            " \u0c05\u0c02\u0c26\u0c41\u0c35\u0c32\u0c4d\u0c32 "
            "\u0c05\u0c24\u0c21\u0c3f \u0c35\u0c26\u0c4d\u0c26 58-23=35 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c05\u0c24\u0c21\u0c41 \u0c2e\u0c30\u0c4b 2 "
            "\u0c15\u0c4b\u0c32\u0c4d\u0c2a\u0c4b\u0c2f\u0c3f\u0c28 "
            "\u0c24\u0c30\u0c41\u0c35\u0c3e\u0c24, "
            "\u0c05\u0c24\u0c21\u0c3f \u0c35\u0c26\u0c4d\u0c26 "
            "\u0c07\u0c2a\u0c4d\u0c2a\u0c41\u0c21\u0c41 35-2=33 "
            "\u0c2c\u0c3e\u0c32\u0c4d\u0c38\u0c4d "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02 33.",
        },
        "6": {
            "q": "\u0c2a\u0c4d\u0c30\u0c36\u0c4d\u0c28: "
            "\u0c12\u0c32\u0c40\u0c35\u0c3f\u0c2f\u0c3e "
            "\u0c35\u0c26\u0c4d\u0c26 $23 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c06\u0c2e\u0c46 "
            "\u0c12\u0c15\u0c4d\u0c15\u0c4a\u0c15\u0c4d\u0c15\u0c1f\u0c3f "
            "$3 \u0c35\u0c3f\u0c32\u0c41\u0c35 \u0c1a\u0c47\u0c38\u0c47 "
            "\u0c10\u0c26\u0c41 \u0c17\u0c3e\u0c1c\u0c41\u0c32\u0c41 "
            "\u0c15\u0c4a\u0c28\u0c41\u0c17\u0c4b\u0c32\u0c41 "
            "\u0c1a\u0c47\u0c38\u0c3f\u0c02\u0c26\u0c3f. "
            "\u0c06\u0c2e\u0c46 \u0c35\u0c26\u0c4d\u0c26 "
            "\u0c0e\u0c02\u0c24 \u0c21\u0c2c\u0c4d\u0c2c\u0c41 "
            "\u0c2e\u0c3f\u0c17\u0c3f\u0c32\u0c3f "
            "\u0c09\u0c02\u0c26\u0c3f?",
            "a": "\u0c26\u0c36\u0c32\u0c35\u0c3e\u0c30\u0c40\u0c17\u0c3e "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02: "
            "\u0c12\u0c15\u0c4d\u0c15\u0c4a\u0c15\u0c4d\u0c15\u0c1f\u0c3f "
            "$3 \u0c1a\u0c4a\u0c2a\u0c4d\u0c2a\u0c41\u0c28 5 "
            "\u0c17\u0c3e\u0c1c\u0c41\u0c32 \u0c27\u0c30 5*3=15 "
            "\u0c21\u0c3e\u0c32\u0c30\u0c4d\u0c32\u0c41. "
            "\u0c12\u0c32\u0c40\u0c35\u0c3f\u0c2f\u0c3e "
            "\u0c35\u0c26\u0c4d\u0c26 "
            "\u0c2a\u0c4d\u0c30\u0c3e\u0c30\u0c02\u0c2d\u0c02\u0c32\u0c4b "
            "$23 \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c05\u0c02\u0c26\u0c41\u0c35\u0c32\u0c4d\u0c32 "
            "\u0c07\u0c2a\u0c4d\u0c2a\u0c41\u0c21\u0c41 \u0c06\u0c2e\u0c46"
            " \u0c35\u0c26\u0c4d\u0c26 23-15=8 "
            "\u0c21\u0c3e\u0c32\u0c30\u0c4d\u0c32\u0c41 "
            "\u0c2e\u0c3f\u0c17\u0c3f\u0c32\u0c3f "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02 8.",
        },
        "7": {
            "q": "\u0c2a\u0c4d\u0c30\u0c36\u0c4d\u0c28: "
            "\u0c1c\u0c47\u0c38\u0c28\u0c4d \u0c35\u0c26\u0c4d\u0c26 20 "
            "\u0c32\u0c3e\u0c32\u0c40\u0c2a\u0c3e\u0c2a\u0c4d\u200c\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c05\u0c24\u0c21\u0c41 "
            "\u0c21\u0c46\u0c28\u0c4d\u0c28\u0c40\u0c15\u0c3f "
            "\u0c15\u0c4a\u0c28\u0c4d\u0c28\u0c3f "
            "\u0c32\u0c3e\u0c32\u0c40\u0c2a\u0c3e\u0c2a\u0c4d\u200c\u0c32\u0c41"
            " \u0c07\u0c1a\u0c4d\u0c1a\u0c3e\u0c21\u0c41. "
            "\u0c07\u0c2a\u0c4d\u0c2a\u0c41\u0c21\u0c41 "
            "\u0c1c\u0c47\u0c38\u0c28\u0c4d \u0c35\u0c26\u0c4d\u0c26 12 "
            "\u0c32\u0c3e\u0c32\u0c40\u0c2a\u0c3e\u0c2a\u0c4d\u200c\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c21\u0c46\u0c28\u0c4d\u0c28\u0c40\u0c15\u0c3f "
            "\u0c1c\u0c47\u0c38\u0c28\u0c4d \u0c0e\u0c28\u0c4d\u0c28\u0c3f"
            " "
            "\u0c32\u0c3e\u0c32\u0c40\u0c2a\u0c3e\u0c2a\u0c4d\u200c\u0c32\u0c41"
            " \u0c07\u0c1a\u0c4d\u0c1a\u0c3e\u0c21\u0c41?"
            "\u0c32\u0c3e\u0c32\u0c40\u0c2a\u0c3e\u0c2a\u0c4d\u200c\u0c32\u0c41"
            "\u0c32\u0c3e\u0c32\u0c40\u0c2a\u0c3e\u0c2a\u0c4d\u200c\u0c32\u0c41"
            "\u0c32\u0c3e\u0c32\u0c40\u0c2a\u0c3e\u0c2a\u0c4d\u200c\u0c32\u0c41",
            "a": "\u0c26\u0c36\u0c32\u0c35\u0c3e\u0c30\u0c40\u0c17\u0c3e "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02: "
            "\u0c1c\u0c47\u0c38\u0c28\u0c4d 20 "
            "\u0c32\u0c3e\u0c32\u0c40\u0c2a\u0c3e\u0c2a\u0c4d\u200c\u0c32\u0c24\u0c4b"
            " "
            "\u0c2a\u0c4d\u0c30\u0c3e\u0c30\u0c02\u0c2d\u0c3f\u0c02\u0c1a\u0c3e\u0c21\u0c41,"
            " \u0c15\u0c3e\u0c28\u0c40 "
            "\u0c07\u0c2a\u0c4d\u0c2a\u0c41\u0c21\u0c41 "
            "\u0c05\u0c24\u0c21\u0c3f \u0c35\u0c26\u0c4d\u0c26 12 "
            "\u0c2e\u0c3e\u0c24\u0c4d\u0c30\u0c2e\u0c47 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f, "
            "\u0c05\u0c02\u0c26\u0c41\u0c35\u0c32\u0c4d\u0c32 "
            "\u0c05\u0c24\u0c21\u0c41 "
            "\u0c21\u0c46\u0c28\u0c4d\u0c28\u0c40\u0c15\u0c3f 20-12=8 "
            "\u0c32\u0c3e\u0c32\u0c40\u0c2a\u0c3e\u0c2a\u0c4d\u200c\u0c32\u0c41"
            " \u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02 8.",
        },
        "8": {
            "q": "\u0c2a\u0c4d\u0c30\u0c36\u0c4d\u0c28: "
            "\u0c2a\u0c3e\u0c30\u0c4d\u0c15\u0c3f\u0c02\u0c17\u0c4d "
            "\u0c32\u0c3e\u0c1f\u0c4d\u200c\u0c32\u0c4b 3 "
            "\u0c15\u0c3e\u0c30\u0c4d\u0c32\u0c41 "
            "\u0c09\u0c02\u0c21\u0c3f, \u0c2e\u0c30\u0c4b 2 "
            "\u0c15\u0c3e\u0c30\u0c4d\u0c32\u0c41 "
            "\u0c35\u0c1a\u0c4d\u0c1a\u0c3f\u0c28\u0c1f\u0c4d\u0c32\u0c2f\u0c3f\u0c24\u0c47,"
            " \u0c2a\u0c3e\u0c30\u0c4d\u0c15\u0c3f\u0c02\u0c17\u0c4d "
            "\u0c32\u0c3e\u0c1f\u0c4d\u200c\u0c32\u0c4b "
            "\u0c0e\u0c28\u0c4d\u0c28\u0c3f "
            "\u0c15\u0c3e\u0c30\u0c4d\u0c32\u0c41 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f?",
            "a": "\u0c26\u0c36\u0c32\u0c35\u0c3e\u0c30\u0c40\u0c17\u0c3e "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02: "
            "\u0c2a\u0c4d\u0c30\u0c3e\u0c30\u0c02\u0c2d\u0c02\u0c32\u0c4b "
            "3 \u0c15\u0c3e\u0c30\u0c4d\u0c32\u0c41 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f, "
            "\u0c2e\u0c30\u0c4b 2 \u0c15\u0c3e\u0c30\u0c4d\u0c32\u0c41 "
            "\u0c35\u0c1a\u0c4d\u0c1a\u0c3e\u0c2f\u0c3f, "
            "\u0c05\u0c02\u0c26\u0c41\u0c35\u0c32\u0c4d\u0c32 "
            "\u0c07\u0c2a\u0c4d\u0c2a\u0c41\u0c21\u0c41 3+2=5 "
            "\u0c15\u0c3e\u0c30\u0c4d\u0c32\u0c41 "
            "\u0c09\u0c28\u0c4d\u0c28\u0c3e\u0c2f\u0c3f. "
            "\u0c38\u0c2e\u0c3e\u0c27\u0c3e\u0c28\u0c02 5.",
        },
    },
    "th": {
        "1": {
            "q": "\u0e42\u0e08\u0e17\u0e22\u0e4c: "
            "\u0e42\u0e23\u0e40\u0e08\u0e2d\u0e23\u0e4c\u0e21\u0e35\u0e25\u0e39\u0e01\u0e40\u0e17\u0e19\u0e19\u0e34\u0e2a"
            " 5 \u0e25\u0e39\u0e01 "
            "\u0e40\u0e02\u0e32\u0e0b\u0e37\u0e49\u0e2d\u0e25\u0e39\u0e01\u0e40\u0e17\u0e19\u0e19\u0e34\u0e2a\u0e40\u0e1e\u0e34\u0e48\u0e21\u0e2d\u0e35\u0e01"
            " 2 \u0e01\u0e23\u0e30\u0e1b\u0e4b\u0e2d\u0e07 "
            "\u0e42\u0e14\u0e22\u0e41\u0e15\u0e48\u0e25\u0e30\u0e01\u0e23\u0e30\u0e1b\u0e4b\u0e2d\u0e07\u0e21\u0e35\u0e25\u0e39\u0e01\u0e40\u0e17\u0e19\u0e19\u0e34\u0e2a"
            " 3 \u0e25\u0e39\u0e01 "
            "\u0e15\u0e2d\u0e19\u0e19\u0e35\u0e49\u0e40\u0e02\u0e32\u0e21\u0e35\u0e25\u0e39\u0e01\u0e40\u0e17\u0e19\u0e19\u0e34\u0e2a\u0e01\u0e35\u0e48\u0e25\u0e39\u0e01",
            "a": "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e17\u0e35\u0e25\u0e30\u0e02\u0e31\u0e49\u0e19\u0e15\u0e2d\u0e19:"
            " "
            "\u0e42\u0e23\u0e40\u0e08\u0e2d\u0e23\u0e4c\u0e40\u0e23\u0e34\u0e48\u0e21\u0e14\u0e49\u0e27\u0e22\u0e01\u0e32\u0e23\u0e21\u0e35\u0e25\u0e39\u0e01\u0e40\u0e17\u0e19\u0e19\u0e34\u0e2a"
            " 5 \u0e25\u0e39\u0e01 "
            "\u0e01\u0e23\u0e30\u0e1b\u0e4b\u0e2d\u0e07 2 "
            "\u0e43\u0e1a\u0e17\u0e35\u0e48\u0e21\u0e35\u0e25\u0e39\u0e01\u0e40\u0e17\u0e19\u0e19\u0e34\u0e2a\u0e43\u0e1a\u0e25\u0e30"
            " 3 \u0e25\u0e39\u0e01 "
            "\u0e40\u0e17\u0e48\u0e32\u0e01\u0e31\u0e1a\u0e21\u0e35\u0e25\u0e39\u0e01\u0e40\u0e17\u0e19\u0e19\u0e34\u0e2a"
            " 6 \u0e25\u0e39\u0e01 "
            "\u0e14\u0e31\u0e07\u0e19\u0e31\u0e49\u0e19 5 + 6 = 11 "
            "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e04\u0e37\u0e2d 11",
        },
        "2": {
            "q": "\u0e42\u0e08\u0e17\u0e22\u0e4c: "
            "\u0e21\u0e35\u0e04\u0e2d\u0e21\u0e1e\u0e34\u0e27\u0e40\u0e15\u0e2d\u0e23\u0e4c\u0e40\u0e01\u0e49\u0e32\u0e40\u0e04\u0e23\u0e37\u0e48\u0e2d\u0e07\u0e43\u0e19\u0e2b\u0e49\u0e2d\u0e07\u0e40\u0e0b\u0e34\u0e23\u0e4c\u0e1f\u0e40\u0e27\u0e2d\u0e23\u0e4c"
            " "
            "\u0e42\u0e14\u0e22\u0e15\u0e31\u0e49\u0e07\u0e41\u0e15\u0e48\u0e27\u0e31\u0e19\u0e08\u0e31\u0e19\u0e17\u0e23\u0e4c\u0e16\u0e36\u0e07\u0e27\u0e31\u0e19\u0e1e\u0e24\u0e2b\u0e31\u0e2a\u0e1a\u0e14\u0e35\u0e21\u0e35\u0e04\u0e2d\u0e21\u0e1e\u0e34\u0e27\u0e40\u0e15\u0e2d\u0e23\u0e4c\u0e15\u0e34\u0e14\u0e15\u0e31\u0e49\u0e07\u0e40\u0e1e\u0e34\u0e48\u0e21\u0e2d\u0e35\u0e01\u0e27\u0e31\u0e19\u0e25\u0e30\u0e2b\u0e49\u0e32\u0e40\u0e04\u0e23\u0e37\u0e48\u0e2d\u0e07"
            " "
            "\u0e15\u0e2d\u0e19\u0e19\u0e35\u0e49\u0e21\u0e35\u0e04\u0e2d\u0e21\u0e1e\u0e34\u0e27\u0e40\u0e15\u0e2d\u0e23\u0e4c\u0e43\u0e19\u0e2b\u0e49\u0e2d\u0e07\u0e40\u0e0b\u0e34\u0e23\u0e4c\u0e1f\u0e40\u0e27\u0e2d\u0e23\u0e4c\u0e01\u0e35\u0e48\u0e40\u0e04\u0e23\u0e37\u0e48\u0e2d\u0e07",
            "a": "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e17\u0e35\u0e25\u0e30\u0e02\u0e31\u0e49\u0e19\u0e15\u0e2d\u0e19:"
            " "
            "\u0e15\u0e31\u0e49\u0e07\u0e41\u0e15\u0e48\u0e27\u0e31\u0e19\u0e08\u0e31\u0e19\u0e17\u0e23\u0e4c\u0e16\u0e36\u0e07\u0e27\u0e31\u0e19\u0e1e\u0e24\u0e2b\u0e31\u0e2a\u0e1a\u0e14\u0e35\u0e23\u0e27\u0e21\u0e01\u0e31\u0e19\u0e40\u0e1b\u0e47\u0e19"
            " 4 \u0e27\u0e31\u0e19 "
            "\u0e42\u0e14\u0e22\u0e41\u0e15\u0e48\u0e25\u0e30\u0e27\u0e31\u0e19\u0e40\u0e1e\u0e34\u0e48\u0e21\u0e04\u0e2d\u0e21\u0e1e\u0e34\u0e27\u0e40\u0e15\u0e2d\u0e23\u0e4c\u0e44\u0e1b"
            " 5 \u0e40\u0e04\u0e23\u0e37\u0e48\u0e2d\u0e07 "
            "\u0e41\u0e2a\u0e14\u0e07\u0e27\u0e48\u0e32\u0e40\u0e1e\u0e34\u0e48\u0e21\u0e04\u0e2d\u0e21\u0e1e\u0e34\u0e27\u0e40\u0e15\u0e2d\u0e23\u0e4c\u0e44\u0e1b\u0e17\u0e31\u0e49\u0e07\u0e2b\u0e21\u0e14"
            " 4 * 5 = 20 \u0e40\u0e04\u0e23\u0e37\u0e48\u0e2d\u0e07 "
            "\u0e41\u0e23\u0e01\u0e40\u0e23\u0e34\u0e48\u0e21\u0e21\u0e35\u0e04\u0e2d\u0e21\u0e1e\u0e34\u0e27\u0e40\u0e15\u0e2d\u0e23\u0e4c\u0e2d\u0e22\u0e39\u0e48"
            " 9 \u0e40\u0e04\u0e23\u0e37\u0e48\u0e2d\u0e07 "
            "\u0e14\u0e31\u0e07\u0e19\u0e31\u0e49\u0e19 "
            "\u0e15\u0e2d\u0e19\u0e19\u0e35\u0e49\u0e08\u0e30\u0e21\u0e35\u0e04\u0e2d\u0e21\u0e1e\u0e34\u0e27\u0e40\u0e15\u0e2d\u0e23\u0e4c\u0e2d\u0e22\u0e39\u0e48"
            " 9 + 20 = 29 \u0e40\u0e04\u0e23\u0e37\u0e48\u0e2d\u0e07 "
            "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e04\u0e37\u0e2d 29",
        },
        "3": {
            "q": "\u0e42\u0e08\u0e17\u0e22\u0e4c: "
            "\u0e25\u0e35\u0e2d\u0e32\u0e21\u0e35\u0e0a\u0e47\u0e2d\u0e01\u0e42\u0e01\u0e41\u0e25\u0e15\u0e2d\u0e22\u0e39\u0e48"
            " 32 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e41\u0e25\u0e30\u0e19\u0e49\u0e2d\u0e07\u0e2a\u0e32\u0e27\u0e21\u0e35\u0e0a\u0e47\u0e2d\u0e01\u0e42\u0e01\u0e41\u0e25\u0e15\u0e2d\u0e22\u0e39\u0e48"
            " 42 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e2b\u0e32\u0e01\u0e17\u0e31\u0e49\u0e07\u0e2a\u0e2d\u0e07\u0e04\u0e19\u0e17\u0e32\u0e19\u0e0a\u0e47\u0e2d\u0e01\u0e42\u0e01\u0e41\u0e25\u0e15\u0e44\u0e1b\u0e41\u0e25\u0e49\u0e27"
            " 35 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e08\u0e30\u0e40\u0e2b\u0e25\u0e37\u0e2d\u0e0a\u0e47\u0e2d\u0e01\u0e42\u0e01\u0e41\u0e25\u0e15\u0e17\u0e31\u0e49\u0e07\u0e2b\u0e21\u0e14\u0e01\u0e35\u0e48\u0e0a\u0e34\u0e49\u0e19",
            "a": "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e17\u0e35\u0e25\u0e30\u0e02\u0e31\u0e49\u0e19\u0e15\u0e2d\u0e19:"
            " "
            "\u0e25\u0e35\u0e2d\u0e32\u0e21\u0e35\u0e0a\u0e47\u0e2d\u0e01\u0e42\u0e01\u0e41\u0e25\u0e15\u0e2d\u0e22\u0e39\u0e48"
            " 32 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e41\u0e25\u0e30\u0e19\u0e49\u0e2d\u0e07\u0e2a\u0e32\u0e27\u0e21\u0e35\u0e0a\u0e47\u0e2d\u0e01\u0e42\u0e01\u0e41\u0e25\u0e15\u0e2d\u0e22\u0e39\u0e48"
            " 42 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e41\u0e2a\u0e14\u0e07\u0e27\u0e48\u0e32\u0e40\u0e14\u0e34\u0e21\u0e21\u0e35\u0e0a\u0e47\u0e2d\u0e01\u0e42\u0e01\u0e41\u0e25\u0e15\u0e2d\u0e22\u0e39\u0e48"
            " 32 + 42 = 74 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e2b\u0e32\u0e01\u0e17\u0e32\u0e19\u0e44\u0e1b\u0e41\u0e25\u0e49\u0e27"
            " 35 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e14\u0e31\u0e07\u0e19\u0e31\u0e49\u0e19\u0e08\u0e30\u0e40\u0e2b\u0e25\u0e37\u0e2d\u0e0a\u0e47\u0e2d\u0e01\u0e42\u0e01\u0e41\u0e25\u0e15\u0e17\u0e31\u0e49\u0e07\u0e2b\u0e21\u0e14\u0e2d\u0e22\u0e39\u0e48"
            " 74 - 35 = 39 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e04\u0e37\u0e2d 39",
        },
        "4": {
            "q": "\u0e42\u0e08\u0e17\u0e22\u0e4c: "
            "\u0e0a\u0e2d\u0e27\u0e4c\u0e19\u0e21\u0e35\u0e02\u0e2d\u0e07\u0e40\u0e25\u0e48\u0e19\u0e2b\u0e49\u0e32\u0e0a\u0e34\u0e49\u0e19"
            " "
            "\u0e43\u0e19\u0e27\u0e31\u0e19\u0e04\u0e23\u0e34\u0e2a\u0e15\u0e4c\u0e21\u0e32\u0e2a"
            " "
            "\u0e40\u0e02\u0e32\u0e44\u0e14\u0e49\u0e23\u0e31\u0e1a\u0e02\u0e2d\u0e07\u0e40\u0e25\u0e48\u0e19\u0e08\u0e32\u0e01\u0e41\u0e21\u0e48\u0e41\u0e25\u0e30\u0e1e\u0e48\u0e2d\u0e04\u0e19\u0e25\u0e30\u0e2a\u0e2d\u0e07\u0e0a\u0e34\u0e49\u0e19"
            " "
            "\u0e15\u0e2d\u0e19\u0e19\u0e35\u0e49\u0e0a\u0e2d\u0e27\u0e4c\u0e19\u0e21\u0e35\u0e02\u0e2d\u0e07\u0e40\u0e25\u0e48\u0e19\u0e01\u0e35\u0e48\u0e0a\u0e34\u0e49\u0e19",
            "a": "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e17\u0e35\u0e25\u0e30\u0e02\u0e31\u0e49\u0e19\u0e15\u0e2d\u0e19:"
            " "
            "\u0e0a\u0e2d\u0e27\u0e4c\u0e19\u0e21\u0e35\u0e02\u0e2d\u0e07\u0e40\u0e25\u0e48\u0e19"
            " 5 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e40\u0e02\u0e32\u0e44\u0e14\u0e49\u0e23\u0e31\u0e1a\u0e02\u0e2d\u0e07\u0e40\u0e25\u0e48\u0e19\u0e08\u0e32\u0e01\u0e41\u0e21\u0e48"
            " 2 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e41\u0e2a\u0e14\u0e07\u0e27\u0e48\u0e32\u0e15\u0e2d\u0e19\u0e19\u0e35\u0e49\u0e0a\u0e2d\u0e27\u0e4c\u0e19\u0e21\u0e35\u0e02\u0e2d\u0e07\u0e40\u0e25\u0e48\u0e19\u0e2d\u0e22\u0e39\u0e48"
            " 5 + 2 = 7 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e41\u0e25\u0e49\u0e27\u0e40\u0e02\u0e32\u0e01\u0e47\u0e44\u0e14\u0e49\u0e23\u0e31\u0e1a\u0e02\u0e2d\u0e07\u0e40\u0e25\u0e48\u0e19\u0e08\u0e32\u0e01\u0e1e\u0e48\u0e2d\u0e40\u0e1e\u0e34\u0e48\u0e21\u0e2d\u0e35\u0e01"
            " 2 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e14\u0e31\u0e07\u0e19\u0e31\u0e49\u0e19 "
            "\u0e0a\u0e2d\u0e27\u0e4c\u0e19\u0e08\u0e36\u0e07\u0e21\u0e35\u0e02\u0e2d\u0e07\u0e40\u0e25\u0e48\u0e19\u0e23\u0e27\u0e21\u0e17\u0e31\u0e49\u0e07\u0e2b\u0e21\u0e14"
            " 7 + 2 = 9 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e04\u0e37\u0e2d 9",
        },
        "5": {
            "q": "\u0e42\u0e08\u0e17\u0e22\u0e4c: "
            "\u0e44\u0e21\u0e40\u0e04\u0e34\u0e25\u0e21\u0e35\u0e25\u0e39\u0e01\u0e01\u0e2d\u0e25\u0e4c\u0e1f"
            " 58 \u0e25\u0e39\u0e01 "
            "\u0e43\u0e19\u0e27\u0e31\u0e19\u0e2d\u0e31\u0e07\u0e04\u0e32\u0e23"
            " "
            "\u0e40\u0e02\u0e32\u0e17\u0e33\u0e25\u0e39\u0e01\u0e01\u0e2d\u0e25\u0e4c\u0e1f\u0e2b\u0e32\u0e22\u0e44\u0e1b"
            " 23 \u0e25\u0e39\u0e01, "
            "\u0e43\u0e19\u0e27\u0e31\u0e19\u0e1e\u0e38\u0e18 "
            "\u0e40\u0e02\u0e32\u0e17\u0e33\u0e2b\u0e32\u0e22\u0e2d\u0e35\u0e01"
            " 2 \u0e25\u0e39\u0e01, "
            "\u0e2a\u0e34\u0e49\u0e19\u0e2a\u0e38\u0e14\u0e27\u0e31\u0e19\u0e1e\u0e38\u0e18"
            " "
            "\u0e44\u0e21\u0e40\u0e04\u0e34\u0e25\u0e21\u0e35\u0e25\u0e39\u0e01\u0e01\u0e2d\u0e25\u0e4c\u0e1f\u0e01\u0e35\u0e48\u0e25\u0e39\u0e01",
            "a": "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e17\u0e35\u0e25\u0e30\u0e02\u0e31\u0e49\u0e19\u0e15\u0e2d\u0e19:"
            " \u0e41\u0e23\u0e01\u0e40\u0e23\u0e34\u0e48\u0e21 "
            "\u0e44\u0e21\u0e40\u0e04\u0e34\u0e25\u0e21\u0e35\u0e25\u0e39\u0e01\u0e01\u0e2d\u0e25\u0e4c\u0e1f\u0e2d\u0e22\u0e39\u0e48"
            " 58 "
            "\u0e25\u0e39\u0e01\u0e41\u0e25\u0e30\u0e17\u0e33\u0e2b\u0e32\u0e22\u0e44\u0e1b"
            " 23 \u0e25\u0e39\u0e01 "
            "\u0e41\u0e2a\u0e14\u0e07\u0e27\u0e48\u0e32\u0e40\u0e02\u0e32\u0e40\u0e2b\u0e25\u0e37\u0e2d\u0e25\u0e39\u0e01\u0e01\u0e2d\u0e25\u0e4c\u0e1f\u0e2d\u0e22\u0e39\u0e48"
            " 58 - 23 = 35 \u0e25\u0e39\u0e01 "
            "\u0e2b\u0e25\u0e31\u0e07\u0e08\u0e32\u0e01\u0e19\u0e31\u0e49\u0e19\u0e40\u0e02\u0e32\u0e17\u0e33\u0e2b\u0e32\u0e22\u0e2d\u0e35\u0e01"
            " 2 \u0e25\u0e39\u0e01 "
            "\u0e14\u0e31\u0e07\u0e19\u0e31\u0e49\u0e19 "
            "\u0e15\u0e2d\u0e19\u0e19\u0e35\u0e49\u0e40\u0e02\u0e32\u0e08\u0e36\u0e07\u0e21\u0e35\u0e25\u0e39\u0e01\u0e01\u0e2d\u0e25\u0e4c\u0e1f\u0e2d\u0e22\u0e39\u0e48"
            " 35 - 2 = 33 \u0e25\u0e39\u0e01 "
            "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e04\u0e37\u0e2d 33",
        },
        "6": {
            "q": "\u0e42\u0e08\u0e17\u0e22\u0e4c: "
            "\u0e42\u0e2d\u0e25\u0e34\u0e40\u0e27\u0e35\u0e22\u0e21\u0e35\u0e40\u0e07\u0e34\u0e19\u0e2d\u0e22\u0e39\u0e48"
            " $23 "
            "\u0e40\u0e18\u0e2d\u0e0b\u0e37\u0e49\u0e2d\u0e02\u0e19\u0e21\u0e1b\u0e31\u0e07\u0e40\u0e1a\u0e40\u0e01\u0e34\u0e25\u0e44\u0e1b\u0e2b\u0e49\u0e32\u0e0a\u0e34\u0e49\u0e19"
            " \u0e23\u0e32\u0e04\u0e32\u0e0a\u0e34\u0e49\u0e19\u0e25\u0e30"
            " $3 "
            "\u0e42\u0e2d\u0e25\u0e34\u0e40\u0e27\u0e35\u0e22\u0e40\u0e2b\u0e25\u0e37\u0e2d\u0e40\u0e07\u0e34\u0e19\u0e2d\u0e22\u0e39\u0e48\u0e40\u0e17\u0e48\u0e32\u0e44\u0e23",
            "a": "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e17\u0e35\u0e25\u0e30\u0e02\u0e31\u0e49\u0e19\u0e15\u0e2d\u0e19:"
            " "
            "\u0e02\u0e19\u0e21\u0e1b\u0e31\u0e07\u0e40\u0e1a\u0e40\u0e01\u0e34\u0e25"
            " 5 \u0e0a\u0e34\u0e49\u0e19 "
            "\u0e23\u0e32\u0e04\u0e32\u0e0a\u0e34\u0e49\u0e19\u0e25\u0e30 "
            "$3 "
            "\u0e23\u0e27\u0e21\u0e40\u0e1b\u0e47\u0e19\u0e40\u0e07\u0e34\u0e19"
            " 5 * 3 = 15 \u0e14\u0e2d\u0e25\u0e25\u0e32\u0e23\u0e4c "
            "\u0e41\u0e23\u0e01\u0e40\u0e23\u0e34\u0e48\u0e21\u0e42\u0e2d\u0e25\u0e34\u0e40\u0e27\u0e35\u0e22\u0e21\u0e35\u0e40\u0e07\u0e34\u0e19\u0e2d\u0e22\u0e39\u0e48"
            " $23 \u0e14\u0e31\u0e07\u0e19\u0e31\u0e49\u0e19 "
            "\u0e15\u0e2d\u0e19\u0e19\u0e35\u0e49\u0e42\u0e2d\u0e25\u0e34\u0e40\u0e27\u0e35\u0e22\u0e08\u0e30\u0e40\u0e2b\u0e25\u0e37\u0e2d\u0e40\u0e07\u0e34\u0e19\u0e2d\u0e22\u0e39\u0e48"
            " 23 - 15 = 8 \u0e14\u0e2d\u0e25\u0e25\u0e32\u0e23\u0e4c "
            "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e04\u0e37\u0e2d 8",
        },
        "7": {
            "q": "\u0e42\u0e08\u0e17\u0e22\u0e4c: "
            "\u0e40\u0e08\u0e2a\u0e31\u0e19\u0e21\u0e35\u0e25\u0e39\u0e01\u0e2d\u0e21\u0e42\u0e25\u0e25\u0e34\u0e1b\u0e4a\u0e2d\u0e1b"
            " 20 \u0e41\u0e17\u0e48\u0e07 "
            "\u0e40\u0e02\u0e32\u0e43\u0e2b\u0e49\u0e41\u0e14\u0e19\u0e19\u0e35\u0e48\u0e44\u0e1b\u0e08\u0e33\u0e19\u0e27\u0e19\u0e2b\u0e19\u0e36\u0e48\u0e07\u0e08\u0e19\u0e15\u0e2d\u0e19\u0e19\u0e35\u0e49\u0e40\u0e2b\u0e25\u0e37\u0e2d\u0e25\u0e39\u0e01\u0e2d\u0e21\u0e42\u0e25\u0e25\u0e34\u0e1b\u0e4a\u0e2d\u0e1b\u0e2d\u0e22\u0e39\u0e48"
            " 12 \u0e41\u0e17\u0e48\u0e07 "
            "\u0e40\u0e08\u0e2a\u0e31\u0e19\u0e43\u0e2b\u0e49\u0e25\u0e39\u0e01\u0e2d\u0e21\u0e42\u0e25\u0e25\u0e34\u0e1b\u0e4a\u0e2d\u0e1b\u0e01\u0e31\u0e1a\u0e41\u0e14\u0e19\u0e19\u0e35\u0e48\u0e44\u0e1b\u0e01\u0e35\u0e48\u0e41\u0e17\u0e48\u0e07",
            "a": "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e17\u0e35\u0e25\u0e30\u0e02\u0e31\u0e49\u0e19\u0e15\u0e2d\u0e19:"
            " "
            "\u0e41\u0e23\u0e01\u0e40\u0e23\u0e34\u0e48\u0e21\u0e40\u0e08\u0e2a\u0e31\u0e19\u0e21\u0e35\u0e25\u0e39\u0e01\u0e2d\u0e21\u0e42\u0e25\u0e25\u0e34\u0e1b\u0e4a\u0e2d\u0e1b\u0e2d\u0e22\u0e39\u0e48"
            " 20 \u0e41\u0e17\u0e48\u0e07 "
            "\u0e41\u0e15\u0e48\u0e15\u0e2d\u0e19\u0e19\u0e35\u0e49\u0e40\u0e02\u0e32\u0e40\u0e2b\u0e25\u0e37\u0e2d\u0e25\u0e39\u0e01\u0e2d\u0e21\u0e42\u0e25\u0e25\u0e34\u0e1b\u0e4a\u0e2d\u0e1b\u0e2d\u0e22\u0e39\u0e48\u0e40\u0e1e\u0e35\u0e22\u0e07"
            " 12 \u0e41\u0e17\u0e48\u0e07 "
            "\u0e14\u0e31\u0e07\u0e19\u0e31\u0e49\u0e19 "
            "\u0e40\u0e08\u0e2a\u0e31\u0e19\u0e43\u0e2b\u0e49\u0e25\u0e39\u0e01\u0e2d\u0e21\u0e42\u0e25\u0e25\u0e34\u0e1b\u0e4a\u0e2d\u0e1b\u0e41\u0e14\u0e19\u0e19\u0e35\u0e48\u0e44\u0e1b"
            " 20 - 12 = 8 \u0e41\u0e17\u0e48\u0e07 "
            "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e04\u0e37\u0e2d 8",
        },
        "8": {
            "q": "\u0e42\u0e08\u0e17\u0e22\u0e4c: "
            "\u0e2b\u0e32\u0e01\u0e17\u0e35\u0e48\u0e08\u0e2d\u0e14\u0e23\u0e16\u0e21\u0e35\u0e23\u0e16\u0e22\u0e19\u0e15\u0e4c\u0e08\u0e2d\u0e14\u0e2d\u0e22\u0e39\u0e48"
            " 3 \u0e04\u0e31\u0e19 "
            "\u0e41\u0e25\u0e30\u0e21\u0e35\u0e23\u0e16\u0e22\u0e19\u0e15\u0e4c\u0e17\u0e35\u0e48\u0e40\u0e1e\u0e34\u0e48\u0e07\u0e02\u0e31\u0e1a\u0e21\u0e32\u0e16\u0e36\u0e07\u0e2d\u0e35\u0e01"
            " 2 \u0e04\u0e31\u0e19 "
            "\u0e17\u0e35\u0e48\u0e08\u0e2d\u0e14\u0e23\u0e16\u0e21\u0e35\u0e23\u0e16\u0e22\u0e19\u0e15\u0e4c\u0e01\u0e35\u0e48\u0e04\u0e31\u0e19",
            "a": "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e17\u0e35\u0e25\u0e30\u0e02\u0e31\u0e49\u0e19\u0e15\u0e2d\u0e19:"
            " "
            "\u0e41\u0e23\u0e01\u0e40\u0e23\u0e34\u0e48\u0e21\u0e21\u0e35\u0e23\u0e16\u0e22\u0e19\u0e15\u0e4c\u0e2d\u0e22\u0e39\u0e48"
            " 3 \u0e04\u0e31\u0e19 "
            "\u0e02\u0e31\u0e1a\u0e21\u0e32\u0e16\u0e36\u0e07\u0e17\u0e35\u0e48\u0e08\u0e2d\u0e14\u0e23\u0e16\u0e2d\u0e35\u0e01"
            " 2 \u0e04\u0e31\u0e19 "
            "\u0e14\u0e31\u0e07\u0e19\u0e31\u0e49\u0e19 "
            "\u0e08\u0e30\u0e21\u0e35\u0e23\u0e16\u0e22\u0e19\u0e15\u0e4c\u0e23\u0e27\u0e21\u0e01\u0e31\u0e19\u0e40\u0e1b\u0e47\u0e19"
            " 3 + 2 = 5 \u0e04\u0e31\u0e19 "
            "\u0e04\u0e33\u0e15\u0e2d\u0e1a\u0e04\u0e37\u0e2d 5",
        },
    },
    "zh": {
        "1": {
            "q": "\u95ee\u9898\uff1a\u7f57\u6770\u6709 5 "
            "\u4e2a\u7f51\u7403\u3002\u4ed6\u53c8\u4e70\u4e86 2 "
            "\u7f50\u7f51\u7403\u3002\u6bcf\u7f50\u6709 3 "
            "\u4e2a\u7f51\u7403\u3002\u4ed6\u73b0\u5728\u6709\u591a\u5c11\u4e2a\u7f51\u7403\uff1f",
            "a": "\u9010\u6b65\u89e3\u7b54\uff1a\u7f57\u6770\u4e00\u5f00\u59cb\u6709"
            " 5 \u4e2a\u7403\u30022 \u7f50\u5404 3 "
            "\u4e2a\u7f51\u7403\u5c31\u662f 6 \u4e2a\u7f51\u7403\u30025 + "
            "6 = 11\u3002\u7b54\u6848\u662f 11\u3002",
        },
        "2": {
            "q": "\u95ee\u9898\uff1a\u670d\u52a1\u5668\u673a\u623f\u91cc\u6709\u4e5d\u53f0\u7535\u8111\u3002\u4ece\u5468\u4e00\u5230\u5468\u56db\uff0c\u6bcf\u5929\u53c8\u5b89\u88c5\u4e86\u4e94\u53f0\u7535\u8111\u3002\u670d\u52a1\u5668\u673a\u623f\u91cc\u73b0\u5728\u6709\u591a\u5c11\u53f0\u7535\u8111\uff1f",
            "a": "\u9010\u6b65\u89e3\u7b54\uff1a\u4ece\u5468\u4e00\u5230\u5468\u56db\u6709"
            " 4 \u5929\u3002\u6bcf\u5929\u589e\u52a0 5 "
            "\u53f0\u7535\u8111\u3002\u8fd9\u610f\u5473\u7740\u4e00\u5171\u589e\u52a0\u4e86"
            " 4 * 5 = 20 \u53f0\u7535\u8111\u3002\u4e00\u5f00\u59cb\u6709 "
            "9 \u53f0\u7535\u8111\uff0c\u6240\u4ee5\u73b0\u5728\u6709 9 + "
            "20 = 29 \u53f0\u7535\u8111\u3002\u7b54\u6848\u662f 29\u3002",
        },
        "3": {
            "q": "\u95ee\u9898\uff1a\u5229\u4e9a\u6709 32 "
            "\u5757\u5de7\u514b\u529b\uff0c\u5979\u59b9\u59b9\u6709 42 "
            "\u5757\u3002\u5982\u679c\u5979\u4eec\u5403\u4e86 35 "
            "\u5757\uff0c\u5979\u4eec\u4e00\u5171\u8fd8\u5269\u4e0b\u591a\u5c11\u5757\uff1f",
            "a": "\u9010\u6b65\u89e3\u7b54\uff1a\u5229\u4e9a\u6709 32 "
            "\u5757\u5de7\u514b\u529b\uff0c\u5229\u4e9a\u7684\u59b9\u59b9\u6709"
            " 42 \u5757\u3002\u8fd9\u610f\u5473\u7740\u539f\u6765\u6709 32"
            " + 42 = 74 \u5757\u5de7\u514b\u529b\u300235 "
            "\u5757\u88ab\u5403\u6389\u4e86\u3002\u6240\u4ee5\u5979\u4eec\u4e00\u5171\u8fd8\u6709"
            " 74 - 35 = 39 "
            "\u5757\u5de7\u514b\u529b\u3002\u7b54\u6848\u662f 39\u3002",
        },
        "4": {
            "q": "\u95ee\u9898\uff1a\u8096\u6069\u6709\u4e94\u4e2a\u73a9\u5177\u3002\u5723\u8bde\u8282\u4ed6\u4ece\u4ed6\u7238\u7238\u5988\u5988\u90a3\u91cc\u5404\u5f97\u5230\u4e86\u4e24\u4e2a\u73a9\u5177\u3002\u4ed6\u73b0\u5728\u6709\u591a\u5c11\u4e2a\u73a9\u5177\uff1f",
            "a": "\u9010\u6b65\u89e3\u7b54\uff1a\u4ed6\u6709 5 "
            "\u4e2a\u73a9\u5177\u3002\u4ed6\u4ece\u5988\u5988\u90a3\u91cc\u5f97\u5230"
            " 2 \u4e2a\uff0c\u6240\u4ee5\u4e4b\u540e\u4ed6\u6709 5 + 2 = 7"
            " "
            "\u4e2a\u73a9\u5177\u3002\u7136\u540e\u4ed6\u53c8\u4ece\u7238\u7238\u90a3\u91cc\u5f97\u5230\u4e86"
            " 2 \u4e2a\uff0c\u6240\u4ee5\u4ed6\u4e00\u5171\u6709 7 + 2 = 9"
            " \u4e2a\u73a9\u5177\u3002\u7b54\u6848\u662f 9\u3002",
        },
        "5": {
            "q": "\u95ee\u9898\uff1a\u8fc8\u514b\u5c14\u6709 58 "
            "\u4e2a\u9ad8\u5c14\u592b\u7403\u3002\u5468\u4e8c\uff0c\u4ed6\u4e22\u5931\u4e86"
            " 23 "
            "\u4e2a\u9ad8\u5c14\u592b\u7403\u3002\u5468\u4e09\uff0c\u4ed6\u53c8\u4e22\u5931\u4e86"
            " 2 "
            "\u4e2a\u3002\u5468\u4e09\u7ed3\u675f\u65f6\u4ed6\u6709\u591a\u5c11\u4e2a\u9ad8\u5c14\u592b\u7403\uff1f",
            "a": "\u9010\u6b65\u89e3\u7b54\uff1a\u8fc8\u514b\u5c14\u4e00\u5f00\u59cb\u6709"
            " 58 \u4e2a\u9ad8\u5c14\u592b\u7403\uff0c\u4e22\u5931\u4e86 23"
            " \u4e2a\uff0c\u6240\u4ee5\u4ed6\u6709 58 - 23 = 35 "
            "\u4e2a\u3002\u5728\u4ed6\u53c8\u4e22\u5931\u4e86 2 "
            "\u4e2a\u540e\uff0c\u4ed6\u73b0\u5728\u6709 35 - 2 = 33 "
            "\u4e2a\u7403\u3002\u7b54\u6848\u662f 33\u3002",
        },
        "6": {
            "q": "\u95ee\u9898\uff1a\u5965\u5229\u7ef4\u4e9a\u6709 23 "
            "\u7f8e\u5143\u3002\u5979\u4e70\u4e86\u4e94\u4e2a\u5355\u4ef7 "
            "3 "
            "\u7f8e\u5143\u7684\u767e\u5409\u997c\u3002\u5979\u8fd8\u5269\u591a\u5c11\u94b1\uff1f",
            "a": "\u9010\u6b65\u89e3\u7b54\uff1a5 \u4e2a\u5355\u4ef7 3 "
            "\u7f8e\u5143\u7684\u767e\u5409\u997c\u5e94\u8be5\u82b1\u8d39 "
            "5 * 3 = 15 "
            "\u7f8e\u5143\u3002\u5965\u5229\u7ef4\u4e9a\u4e00\u5f00\u59cb\u6709"
            " 23 "
            "\u7f8e\u5143\uff0c\u6240\u4ee5\u73b0\u5728\u5979\u8fd8\u5269 "
            "23 - 15 = 8 \u7f8e\u5143\u3002\u7b54\u6848\u662f 8\u3002",
        },
        "7": {
            "q": "\u95ee\u9898\uff1a\u6770\u68ee\u6709 20 "
            "\u6839\u68d2\u68d2\u7cd6\u3002\u4ed6\u7ed9\u4e86\u4e39\u5c3c\u4e00\u4e9b\u68d2\u68d2\u7cd6\u3002\u73b0\u5728\u6770\u68ee\u6709"
            " 12 "
            "\u6839\u68d2\u68d2\u7cd6\u3002\u6770\u68ee\u7ed9\u4e86\u4e39\u5c3c\u591a\u5c11\u6839\u68d2\u68d2\u7cd6\uff1f",
            "a": "\u9010\u6b65\u89e3\u7b54\uff1a\u6770\u68ee\u4e00\u5f00\u59cb\u6709"
            " 20 "
            "\u6839\u68d2\u68d2\u7cd6\uff0c\u4f46\u73b0\u5728\u4ed6\u53ea\u6709"
            " 12 "
            "\u6839\u4e86\uff0c\u6240\u4ee5\u4ed6\u7ed9\u4e86\u4e39\u5c3c "
            "20 - 12 = 8 \u6839\u68d2\u68d2\u7cd6\u3002\u7b54\u6848\u662f "
            "8\u3002",
        },
        "8": {
            "q": "\u95ee\u9898\uff1a\u5982\u679c\u505c\u8f66\u573a\u91cc\u6709 "
            "3 \u8f86\u8f66\uff0c\u53c8\u6765\u4e86 2 "
            "\u8f86\u8f66\uff0c\u505c\u8f66\u573a\u91cc\u6709\u591a\u5c11\u8f86\u8f66\uff1f",
            "a": "\u9010\u6b65\u89e3\u7b54\uff1a\u4e00\u5f00\u59cb\u6709 3 "
            "\u8f86\u8f66\uff0c\u53c8\u6765\u4e86 2 "
            "\u8f86\uff0c\u6240\u4ee5\u73b0\u5728\u5e94\u8be5\u6709 3 + 2 "
            "= 5 \u8f86\u8f66\u3002\u7b54\u6848\u662f 5\u3002",
        },
    },
}

evals/build_data/mgsm_prompt/exemplars.py
GPT-Fathom/GPT-Fathom
"""
This file defines the `oaieval` CLI for running evals.
"""
import argparse
import logging
import shlex
import sys
from typing import Any, Mapping, Optional, Union, cast

import openai

import evals
import evals.api
import evals.base
import evals.record
from evals.eval import Eval
from evals.registry import Registry

logger = logging.getLogger(__name__)


def _purple(str: str) -> str:
    return f"\033[1;35m{str}\033[0m"


def get_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="Run evals through the API")
    parser.add_argument(
        "completion_fn",
        type=str,
        help="One or more CompletionFn URLs, separated by commas (,). A CompletionFn can either be the name of a model available in the OpenAI API or a key in the registry (see evals/registry/completion_fns).",
    )
    parser.add_argument("eval", type=str, help="Name of an eval. See registry.")
    parser.add_argument("--extra_eval_params", type=str, default="")
    parser.add_argument("--max_samples", type=int, default=None)
    parser.add_argument("--cache", action=argparse.BooleanOptionalAction, default=True)
    parser.add_argument(
        "--visible", action=argparse.BooleanOptionalAction, default=None
    )
    parser.add_argument("--seed", type=int, default=20220722)
    parser.add_argument("--user", type=str, default="")
    parser.add_argument("--record_path", type=str, default=None)
    parser.add_argument(
        "--log_to_file", type=str, default=None, help="Log to a file instead of stdout"
    )
    parser.add_argument(
        "--registry_path",
        type=str,
        default=None,
        action="append",
        help="Path to the registry",
    )
    parser.add_argument(
        "--eval_in_batch", type=bool, help="evaluation is done in batch", default=False
    )
    parser.add_argument(
        "--local_dataset",
        type=bool,
        help="use local dataset for evaluation",
        default=False,
    )
    parser.add_argument(
        "--azure_eval",
        type=bool,
        help="whether or not use azure api in openai eval",
        default=False,
    )
    parser.add_argument("--debug", action=argparse.BooleanOptionalAction, default=False)
    parser.add_argument(
        "--local-run", action=argparse.BooleanOptionalAction, default=True
    )
    parser.add_argument(
        "--dry-run", action=argparse.BooleanOptionalAction, default=False
    )
    parser.add_argument(
        "--dry-run-logging", action=argparse.BooleanOptionalAction, default=True
    )

    return parser


class OaiEvalArguments(argparse.Namespace):
    completion_fn: str
    eval: str
    extra_eval_params: str
    max_samples: Optional[int]
    cache: bool
    visible: Optional[bool]
    seed: int
    user: str
    record_path: Optional[str]
    log_to_file: Optional[str]
    registry_path: Optional[str]
    debug: bool
    local_run: bool
    dry_run: bool
    dry_run_logging: bool
    eval_in_batch: bool
    azure_eval: bool
    local_dataset: bool


def run(args: OaiEvalArguments, registry: Optional[Registry] = None) -> str:
    if args.debug:
        logging.getLogger().setLevel(logging.DEBUG)

    visible = args.visible if args.visible is not None else (args.max_samples is None)

    if args.max_samples is not None:
        evals.eval.set_max_samples(args.max_samples)

    registry = registry or Registry()
    if args.registry_path:
        registry.add_registry_paths(args.registry_path)

    eval_spec = registry.get_eval(args.eval)
    assert (
        eval_spec is not None
    ), f"Eval {args.eval} not found. Available: {list(sorted(registry._evals.keys()))}"

    completion_fns = args.completion_fn.split(",")
    completion_fn_instances = [
        registry.make_completion_fn(args.azure_eval, url) for url in completion_fns
    ]

    run_config = {
        "completion_fns": completion_fns,
        "eval_spec": eval_spec,
        "seed": args.seed,
        "max_samples": args.max_samples,
        "command": " ".join(map(shlex.quote, sys.argv)),
        "initial_settings": {
            "visible": visible,
        },
    }

    eval_name = eval_spec.key
    if eval_name is None:
        raise Exception("you must provide a eval name")

    run_spec = evals.base.RunSpec(
        completion_fns=completion_fns,
        eval_name=eval_name,
        base_eval=eval_name.split(".")[0],
        split=eval_name.split(".")[1],
        run_config=run_config,
        created_by=args.user,
        setting_name=args.eval,
    )
    if args.record_path is None:
        record_path = f"/tmp/evallogs/{run_spec.run_id}_{args.completion_fn}_{args.eval}.jsonl"
    else:
        record_path = args.record_path

    recorder: evals.record.RecorderBase
    if args.dry_run:
        recorder = evals.record.DummyRecorder(
            run_spec=run_spec, log=args.dry_run_logging
        )
    elif args.local_run:
        recorder = evals.record.LocalRecorder(record_path, run_spec=run_spec)
    else:
        recorder = evals.record.Recorder(record_path, run_spec=run_spec)

    api_extra_options: dict[str, Any] = {}
    if not args.cache:
        api_extra_options["cache_level"] = 0

    run_url = f"{run_spec.run_id}"
    logger.info(_purple(f"Run started: {run_url}"))

    def parse_extra_eval_params(
        param_str: Optional[str],
    ) -> Mapping[str, Union[str, int, float]]:
        """Parse a string of the form "key1=value1,key2=value2" into a dict."""
        if not param_str:
            return {}

        def to_number(x: str) -> Union[int, float, str]:
            try:
                return int(x)
            except:
                pass
            try:
                return float(x)
            except:
                pass
            return x

        str_dict = dict(kv.split("=") for kv in param_str.split(","))
        return {k: to_number(v) for k, v in str_dict.items()}

    extra_eval_params = parse_extra_eval_params(args.extra_eval_params)

    eval_class = registry.get_class(eval_spec)
    eval: Eval = eval_class(
        completion_fns=completion_fn_instances,
        seed=args.seed,
        name=eval_name,
        registry=registry,
        eval_in_batch=args.eval_in_batch,
        local_dataset=args.local_dataset,
        **extra_eval_params,
    )
    result = eval.run(recorder)
    recorder.record_final_report(result)

    if not (args.dry_run or args.local_run):
        logger.info(_purple(f"Run completed: {run_url}"))

    logger.info("Final report:")
    for key, value in result.items():
        logger.info(f"{key}: {value}")
    return run_spec.run_id


def main() -> None:
    parser = get_parser()
    args = cast(OaiEvalArguments, parser.parse_args(sys.argv[1:]))
    logging.basicConfig(
        format="[%(asctime)s] [%(filename)s:%(lineno)d] %(message)s",
        level=logging.INFO,
        filename=args.log_to_file if args.log_to_file else None,
    )
    logging.getLogger("openai").setLevel(logging.WARN)

    # TODO)) why do we need this?
    if hasattr(openai.error, "set_display_cause"):  # type: ignore
        openai.error.set_display_cause()  # type: ignore
    run(args)


if __name__ == "__main__":
    main()

evals/cli/oaieval.py
GPT-Fathom/GPT-Fathom
"""
This file defines the `oaievalset` CLI for running eval sets.
"""
import argparse
import json
import logging
import subprocess
from pathlib import Path
from typing import Optional, cast

from evals.registry import Registry

Task = list[str]
logger = logging.getLogger(__name__)


class Progress:
    def __init__(self, file: str) -> None:
        self.file = Path(file)
        self.completed: list[Task] = []

    def load(self) -> bool:
        if not self.file.exists():
            return False

        with self.file.open() as f:
            for line in f:
                self.completed.append(json.loads(line))
        return len(self.completed) > 0

    def add(self, item: Task) -> None:
        self.completed.append(item)
        self.save()

    def save(self) -> None:
        self.file.parent.mkdir(parents=True, exist_ok=True)
        with self.file.open("w") as f:
            for item in self.completed:
                f.write(json.dumps(item) + "\n")
            print(highlight(f"Saved progress to {self.file}"))


def highlight(str: str) -> str:
    return f"\033[1;32m>>> {str}\033[0m"


def get_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="Run eval sets through the API")
    parser.add_argument("model", type=str, help="Name of a completion model.")
    parser.add_argument("eval_set", type=str, help="Name of eval set. See registry.")
    parser.add_argument(
        "--resume",
        action=argparse.BooleanOptionalAction,
        default=True,
        help="Resume from last checkpoint.",
    )
    parser.add_argument(
        "--exit-on-error",
        action=argparse.BooleanOptionalAction,
        default=True,
        help="Exit if any oaieval command fails.",
    )
    return parser


class OaiEvalSetArguments(argparse.Namespace):
    model: str
    eval_set: str
    resume: bool
    exit_on_error: bool


def run(
    args: OaiEvalSetArguments,
    unknown_args: list[str],
    registry: Optional[Registry] = None,
    run_command: str = "oaieval",
) -> None:
    registry = registry or Registry()
    commands: list[Task] = []
    eval_set = registry.get_eval_set(args.eval_set) if args.eval_set else None
    if eval_set:
        # print(eval_set.evals)

        for index, eval in enumerate(registry.get_evals(eval_set.evals)):
            # print(index)
            if not eval or not eval.key:
                logger.debug("The eval #%d in eval_set is not valid", index)
            command = [run_command, args.model, eval_set.evals[index]] + unknown_args
            if command in commands:
                continue
            commands.append(command)
    else:
        logger.warning("No eval set found for %s", args.eval_set)

    num_evals = len(commands)

    progress = Progress(
        f"/tmp/oaievalset/{args.model}.{args.eval_set}.progress.txt"
    )
    if args.resume and progress.load():
        print(f"Loaded progress from {progress.file}")
        print(f"{len(progress.completed)}/{len(commands)} evals already completed:")
        for item in progress.completed:
            print("  " + " ".join(item))

    commands = [c for c in commands if c not in progress.completed]
    command_strs = [" ".join(cmd) for cmd in commands]
    print("Going to run the following commands:")
    for command_str in command_strs:
        print("  " + command_str)

    num_already_completed = num_evals - len(commands)
    for idx, command in enumerate(commands):
        real_idx = idx + num_already_completed
        print(
            highlight(
                "Running command: " + " ".join(command) + f" ({real_idx+1}/{num_evals})"
            )
        )
        subprocess.run(command, check=args.exit_on_error)
        progress.add(command)

    print(highlight("All done!"))


def main() -> None:
    parser = get_parser()
    args, unknown_args = parser.parse_known_args()
    run(cast(OaiEvalSetArguments, args), unknown_args)


if __name__ == "__main__":
    main()

evals/cli/oaievalset.py
GPT-Fathom/GPT-Fathom
"""
Extending Completion Functions with Chain-of-Thought
"""
from evals.api import CompletionFn, CompletionResult
from evals.prompt.base import ChatCompletionPrompt
from evals.record import record_sampling
from evals.registry import Registry

DEFAULT_COT_TEMPLATE = "\nBefore answering, reason in a step-by-step manner as to get the right answer, then conclude with the answer."
DEFAULT_EXTRACT_ANSWER_TEMPLATE = "\nGiven the above reasoning, the answer in the format requested by the question is:"


class ChainOfThoughtCompletionResult(CompletionResult):
    def __init__(self, response) -> None:
        self.response = response

    def get_completions(self) -> list[str]:
        return [self.response.strip()]


class ChainOfThoughtCompletionFn(CompletionFn):
    def __init__(
        self,
        cot_template: str = DEFAULT_COT_TEMPLATE,
        extract_answer_template: str = DEFAULT_EXTRACT_ANSWER_TEMPLATE,
        cot_completion_fn: str = None,
        extract_completion_fn: str = None,
        registry: Registry = None,
        registry_path: str = None,
        **kwargs
    ) -> None:
        registry = Registry() if not registry else registry
        if registry_path:
            registry.add_registry_paths(registry_path)

        if extract_completion_fn is None:
            extract_completion_fn = cot_completion_fn

        # This model will use chain of thought to answer the question
        self.cot_template = cot_template
        self.cot_completion_fn_instance = registry.make_completion_fn(cot_completion_fn)

        # This model will extract the answer from the chain of thought
        self.extract_answer_template = extract_answer_template
        self.extract_completion_fn_instance = registry.make_completion_fn(
            extract_completion_fn
        )

    def __call__(self, prompt, **kwargs) -> ChainOfThoughtCompletionResult:
        # Ensure it is in string format
        prompt = ChatCompletionPrompt(prompt).to_formatted_prompt()

        cot_prompt = prompt + [{"role": "assistant", "content": self.cot_template}]
        answer = self.cot_completion_fn_instance(
            prompt=cot_prompt, **kwargs
        ).get_completions()[0]
        record_sampling(prompt=cot_prompt, sampled=answer)

        extraction_prompt = cot_prompt + [
            {"role": "assistant", "content": answer},
            {"role": "assistant", "content": self.extract_answer_template},
        ]
        extracted_answer = self.extract_completion_fn_instance(
            prompt=extraction_prompt, **kwargs
        ).get_completions()[0]
        record_sampling(prompt=extraction_prompt, sampled=extracted_answer)

        return ChainOfThoughtCompletionResult(extracted_answer)

evals/completion_fns/cot.py
GPT-Fathom/GPT-Fathom
import importlib
from typing import Optional

from langchain.llms import BaseLLM

from evals.api import CompletionFn, CompletionResult
from evals.prompt.base import CompletionPrompt
from evals.record import record_sampling


class LangChainLLMCompletionResult(CompletionResult):
    def __init__(self, response) -> None:
        self.response = response

    def get_completions(self) -> list[str]:
        return [self.response.strip()]


class LangChainLLMCompletionFn(CompletionFn):
    def __init__(self, llm: str, llm_kwargs: Optional[dict] = {}, **kwargs) -> None:
        # Import and resolve self.llm to an instance of llm argument here, assuming it's always a subclass of BaseLLM
        module = importlib.import_module("langchain.llms")
        LLMClass = getattr(module, llm)

        if issubclass(LLMClass, BaseLLM):
            self.llm = LLMClass(**llm_kwargs)
        else:
            raise ValueError(f"{llm} is not a subclass of BaseLLM")

    def __call__(self, prompt, **kwargs) -> LangChainLLMCompletionResult:
        prompt = CompletionPrompt(prompt).to_formatted_prompt()
        response = self.llm(prompt)
        record_sampling(prompt=prompt, sampled=response)
        return LangChainLLMCompletionResult(response)

evals/completion_fns/langchain_llm.py
GPT-Fathom/GPT-Fathom
import importlib
from typing import Optional

from langchain import LLMMathChain, OpenAI
from openai import Completion

from evals.api import CompletionResult
from evals.prompt.base import CompletionPrompt
from evals.record import record_sampling


class LangChainCompletionResult(CompletionResult):
    def __init__(self, response) -> None:
        self.response = response

    def get_completions(self) -> list[str]:
        return [self.response.strip()]


class LangChainMathChainCompletionFn(Completion):
    def __init__(self, **kwargs) -> None:
        llm = OpenAI(temperature=0)
        self.llm_math = LLMMathChain(llm=llm)

    def __call__(self, prompt, **kwargs) -> LangChainCompletionResult:

        prompt = CompletionPrompt(prompt).to_formatted_prompt()
        response = self.llm_math.run(prompt)
        # The LangChain response comes with `Answer: ` ahead of this, let's strip it out
        response = response.strip("Answer:").strip()
        record_sampling(prompt=prompt, sampled=response)
        return LangChainCompletionResult(response)

evals/completion_fns/langchain_math.py
GPT-Fathom/GPT-Fathom
from typing import Any, Optional, Union

import torch

from evals.api import CompletionFn, CompletionResult
from evals.base import CompletionFnSpec
from evals.prompt.base import (
    ChatCompletionPrompt,
    CompletionPrompt,
    OpenAICreateChatPrompt,
    OpenAICreatePrompt,
    Prompt,
    chat_prompt_to_text_prompt,
)
from evals.record import record_sampling
from evals.utils.api_utils import (
    openai_chat_completion_create_retrying,
    openai_completion_create_retrying,
)


class OpenAIBaseCompletionResult(CompletionResult):
    def __init__(self, raw_data: Any, prompt: Any):
        self.raw_data = raw_data
        self.prompt = prompt

    def get_completions(self) -> list[str]:
        raise NotImplementedError


class OpenAICompletionResult(OpenAIBaseCompletionResult):
    def get_completions(self) -> list[str]:
        completions = []
        if self.raw_data and "choices" in self.raw_data:
            for choice in self.raw_data["choices"]:
                if "text" in choice:
                    completions.append(choice["text"])
        return completions


class LLAMACompletion(CompletionFn):
    def __init__(
        self,
        model: Optional[str] = None,
        api_base: Optional[str] = None,
        api_key: Optional[str] = None,
        n_ctx: Optional[int] = None,
        extra_options: Optional[dict] = {},
        **kwargs,
    ):
        self.model = "llama"
        self.api_base = api_base
        self.api_key = api_key
        self.n_ctx = n_ctx
        self.extra_options = extra_options

    def __call__(
        self,
        inputs,
        **kwargs,
    ) -> OpenAICompletionResult:
        from tqdm import tqdm
        from transformers import LlamaForCausalLM, LlamaTokenizer

        batch_size = 32
        text = [sample for sample in inputs]

        num_batches = len(text) // batch_size + 1
        # print(len(text))
        llama_path = None
        assert llama_path!=None, "Fix your llama path here"

        tokenizer = LlamaTokenizer.from_pretrained(
            llama_path, use_fast=False, padding_side="left"
        )
        model = LlamaForCausalLM.from_pretrained(
            llama_path,
            low_cpu_mem_usage=True,
            torch_dtype=torch.float16,
        )
        n_gpus = torch.cuda.device_count()
        import tensor_parallel as tp

        # model = tp.tensor_parallel(model, [i for i in range(n_gpus)])
        model = tp.tensor_parallel(model, [i for i in range(n_gpus)])
        tokenizer.pad_token_id = (
            0 if tokenizer.pad_token_id is None else tokenizer.pad_token_id
        )
        tokenizer.bos_token_id = 1

        model.eval()
        # print(num_batches)
        res = []
        for i in tqdm(range(num_batches)):
            batch = text[i * batch_size : min((i + 1) * batch_size, len(text))]
            if len(batch) == 0:
                continue
            input_texts = batch
            inputs = tokenizer.batch_encode_plus(
                input_texts, return_tensors="pt", padding=True
            ).to("cuda")
            with torch.no_grad():
                if "do_sample" not in kwargs.keys():
                    generate_ids = model.generate(
                        **inputs,
                        max_new_tokens=kwargs["max_tokens"],
                        temperature=kwargs.get("temperature", 0.0),
                    )
                else:
                    generate_ids = model.generate(
                        **inputs,
                        max_new_tokens=kwargs["max_tokens"],
                        temperature=kwargs.get("temperature", 0.0),
                        do_sample=kwargs.get("do_sample", 0.0),
                        top_p=kwargs.get("top_p", 0.95),
                    )
            result = tokenizer.batch_decode(
                generate_ids,
                skip_special_tokens=True,
                clean_up_tokenization_spaces=False,
            )
            res.extend(result)

        for i in range(len(res)):
            res[i] = res[i][len(text[i]) :]

        return res


class LLAMA2Completion(CompletionFn):
    def __init__(
        self,
        model: Optional[str] = None,
        api_base: Optional[str] = None,
        api_key: Optional[str] = None,
        n_ctx: Optional[int] = None,
        extra_options: Optional[dict] = {},
        **kwargs,
    ):
        self.model = "llama2"
        self.api_base = api_base
        self.api_key = api_key
        self.n_ctx = n_ctx
        self.extra_options = extra_options

    def __call__(
        self,
        inputs,
        **kwargs,
    ) -> OpenAICompletionResult:
        from tqdm import tqdm
        from transformers import LlamaForCausalLM, LlamaTokenizer

        batch_size = 32
        text = [sample for sample in inputs]

        num_batches = len(text) // batch_size + 1
        max_len = max([len(item) for item in text])

        tokenizer = LlamaTokenizer.from_pretrained(
            "meta-llama/Llama-2-70b-hf",
            use_fast=False,
            padding_side="left",
        )
        model = LlamaForCausalLM.from_pretrained(
            "meta-llama/Llama-2-70b-hf",
            low_cpu_mem_usage=True,
            torch_dtype=torch.float16,
        )
        n_gpus = torch.cuda.device_count()
        import tensor_parallel as tp

        # model = tp.tensor_parallel(model, [i for i in range(n_gpus)])
        model = tp.tensor_parallel(model, [i for i in range(n_gpus)])
        tokenizer.pad_token_id = (
            0 if tokenizer.pad_token_id is None else tokenizer.pad_token_id
        )
        tokenizer.bos_token_id = 1

        model.eval()

        res = []
        for i in tqdm(range(num_batches)):
            batch = text[i * batch_size : min((i + 1) * batch_size, len(text))]
            if len(batch) == 0:
                continue
            input_texts = batch
            inputs = tokenizer.batch_encode_plus(
                input_texts, return_tensors="pt", padding=True
            ).to("cuda")
            with torch.no_grad():
                generate_ids = model.generate(
                    **inputs,
                    max_new_tokens=kwargs["max_tokens"],
                    temperature=kwargs.get("temperature", 0.0),
                )
            result = tokenizer.batch_decode(
                generate_ids,
                skip_special_tokens=True,
                clean_up_tokenization_spaces=False,
            )
            res.extend(result)

        for i in range(len(res)):
            res[i] = res[i][len(text[i]) :]

        return res

evals/completion_fns/llama.py
GPT-Fathom/GPT-Fathom
from typing import Any, Optional, Union

from evals.api import CompletionFn, CompletionResult
from evals.base import CompletionFnSpec
from evals.prompt.base import (
    ChatCompletionPrompt,
    CompletionPrompt,
    OpenAICreateChatPrompt,
    OpenAICreatePrompt,
    Prompt,
)
from evals.record import record_sampling
from evals.utils.api_utils import (
    openai_chat_completion_create_retrying,
    openai_completion_create_retrying,
)
from evals.utils.azure_utils import (
    azure_chat_completion_create_retrying,
    azure_completion_create_retrying,
)


class OpenAIBaseCompletionResult(CompletionResult):
    def __init__(self, raw_data: Any, prompt: Any):
        self.raw_data = raw_data
        self.prompt = prompt

    def get_completions(self) -> list[str]:
        raise NotImplementedError


class OpenAIChatCompletionResult(OpenAIBaseCompletionResult):
    def get_completions(self) -> list[str]:
        completions = []
        if self.raw_data and "choices" in self.raw_data:
            for choice in self.raw_data["choices"]:
                if "message" in choice:
                    completions.append(choice["message"]["content"])
        return completions


class OpenAICompletionResult(OpenAIBaseCompletionResult):
    def get_completions(self) -> list[str]:
        completions = []
        if self.raw_data and "choices" in self.raw_data:
            for choice in self.raw_data["choices"]:
                if "text" in choice:
                    completions.append(choice["text"])
        return completions


class OpenAICompletionFn(CompletionFn):
    def __init__(
        self,
        model: Optional[str] = None,
        api_base: Optional[str] = None,
        api_key: Optional[str] = None,
        n_ctx: Optional[int] = None,
        extra_options: Optional[dict] = {},
        **kwargs,
    ):
        self.model = model
        self.api_base = api_base
        self.api_key = api_key
        self.n_ctx = n_ctx
        self.extra_options = extra_options

    def __call__(
        self,
        prompt: Union[str, OpenAICreateChatPrompt],
        **kwargs,
    ) -> OpenAICompletionResult:
        if not isinstance(prompt, Prompt):
            assert (
                isinstance(prompt, str)
                or (
                    isinstance(prompt, list)
                    and all(isinstance(token, int) for token in prompt)
                )
                or (
                    isinstance(prompt, list)
                    and all(isinstance(token, str) for token in prompt)
                )
                or (
                    isinstance(prompt, list)
                    and all(isinstance(msg, dict) for msg in prompt)
                )
            ), f"Got type {type(prompt)}, with val {type(prompt[0])} for prompt, expected str or list[int] or list[str] or list[dict[str, str]]"

            prompt = CompletionPrompt(
                raw_prompt=prompt,
            )

        openai_create_prompt: OpenAICreatePrompt = prompt.to_formatted_prompt()

        result = openai_completion_create_retrying(
            model=self.model,
            api_base=self.api_base,
            api_key=self.api_key,
            prompt=openai_create_prompt,
            **{**kwargs, **self.extra_options},
        )
        result = OpenAICompletionResult(raw_data=result, prompt=openai_create_prompt)
        record_sampling(prompt=result.prompt, sampled=result.get_completions())
        return result


class OpenAIChatCompletionFn(CompletionFnSpec):
    def __init__(
        self,
        model: Optional[str] = None,
        api_base: Optional[str] = None,
        api_key: Optional[str] = None,
        n_ctx: Optional[int] = None,
        extra_options: Optional[dict] = {},
        **kwargs,
    ):
        self.model = model
        self.api_base = api_base
        self.api_key = api_key
        self.n_ctx = n_ctx
        self.extra_options = extra_options

    def __call__(
        self,
        prompt: Union[str, OpenAICreateChatPrompt],
        **kwargs,
    ) -> OpenAIChatCompletionResult:
        if not isinstance(prompt, Prompt):
            assert (
                isinstance(prompt, str)
                or (
                    isinstance(prompt, list)
                    and all(isinstance(token, int) for token in prompt)
                )
                or (
                    isinstance(prompt, list)
                    and all(isinstance(token, str) for token in prompt)
                )
                or (
                    isinstance(prompt, list)
                    and all(isinstance(msg, dict) for msg in prompt)
                )
            ), f"Got type {type(prompt)}, with val {type(prompt[0])} for prompt, expected str or list[int] or list[str] or list[dict[str, str]]"

            prompt = ChatCompletionPrompt(
                raw_prompt=prompt,
            )

        openai_create_prompt: OpenAICreateChatPrompt = prompt.to_formatted_prompt()

        result = openai_chat_completion_create_retrying(
            model=self.model,
            api_base=self.api_base,
            api_key=self.api_key,
            messages=openai_create_prompt,
            **{**kwargs, **self.extra_options},
        )
        result = OpenAIChatCompletionResult(
            raw_data=result, prompt=openai_create_prompt
        )
        record_sampling(prompt=result.prompt, sampled=result.get_completions())
        return result


MODEL_TO_ENGINE = {
    "Completion Function": "Deployment ID you get From your Azure Deployment",
}


class AzureCompletionFn(CompletionFn):
    def __init__(
        self,
        model: Optional[str] = None,
        api_base: Optional[str] = None,
        api_key: Optional[str] = None,
        n_ctx: Optional[int] = None,
        extra_options: Optional[dict] = {},
        **kwargs,
    ):
        self.model = model
        self.api_base = api_base
        self.api_key = api_key
        self.n_ctx = n_ctx
        self.extra_options = extra_options

    def __call__(
        self,
        prompt: Union[str, OpenAICreateChatPrompt],
        **kwargs,
    ) -> OpenAICompletionResult:
        if not isinstance(prompt, Prompt):
            assert (
                isinstance(prompt, str)
                or (
                    isinstance(prompt, list)
                    and all(isinstance(token, int) for token in prompt)
                )
                or (
                    isinstance(prompt, list)
                    and all(isinstance(token, str) for token in prompt)
                )
                or (
                    isinstance(prompt, list)
                    and all(isinstance(msg, dict) for msg in prompt)
                )
            ), f"Got type {type(prompt)}, with val {type(prompt[0])} for prompt, expected str or list[int] or list[str] or list[dict[str, str]]"

            prompt = CompletionPrompt(
                raw_prompt=prompt,
            )

        openai_create_prompt: OpenAICreatePrompt = prompt.to_formatted_prompt()

        result = azure_completion_create_retrying(
            engine=MODEL_TO_ENGINE[self.model],
            prompt=openai_create_prompt,
            **{**kwargs, **self.extra_options},
        )
        result = OpenAICompletionResult(raw_data=result, prompt=openai_create_prompt)
        record_sampling(prompt=result.prompt, sampled=result.get_completions())
        return result


class AzureChatCompletionFn(CompletionFnSpec):
    def __init__(
        self,
        model: Optional[str] = None,
        api_base: Optional[str] = None,
        api_key: Optional[str] = None,
        n_ctx: Optional[int] = None,
        extra_options: Optional[dict] = {},
        **kwargs,
    ):
        self.model = model
        self.api_base = api_base
        self.api_key = api_key
        self.n_ctx = n_ctx
        self.extra_options = extra_options

    def __call__(
        self,
        prompt: Union[str, OpenAICreateChatPrompt],
        **kwargs,
    ) -> OpenAIChatCompletionResult:
        if not isinstance(prompt, Prompt):
            assert (
                isinstance(prompt, str)
                or (
                    isinstance(prompt, list)
                    and all(isinstance(token, int) for token in prompt)
                )
                or (
                    isinstance(prompt, list)
                    and all(isinstance(token, str) for token in prompt)
                )
                or (
                    isinstance(prompt, list)
                    and all(isinstance(msg, dict) for msg in prompt)
                )
            ), f"Got type {type(prompt)}, with val {type(prompt[0])} for prompt, expected str or list[int] or list[str] or list[dict[str, str]]"

            prompt = ChatCompletionPrompt(
                raw_prompt=prompt,
            )

        openai_create_prompt: OpenAICreateChatPrompt = prompt.to_formatted_prompt()


        result = azure_chat_completion_create_retrying(
            deployment_id=MODEL_TO_ENGINE[self.model],
            messages=openai_create_prompt,
            **{**kwargs, **self.extra_options},
        )
        result = OpenAIChatCompletionResult(
            raw_data=result, prompt=openai_create_prompt
        )
        record_sampling(prompt=result.prompt, sampled=result.get_completions())
        return result

evals/completion_fns/openai.py
GPT-Fathom/GPT-Fathom
"""
Extending Completion Functions with Embeddings-based retrieval from a fetched dataset
"""
from ast import literal_eval
from typing import Any, Optional, Union

import numpy as np
import openai
import pandas as pd

from evals.api import CompletionFn, CompletionResult
from evals.prompt.base import ChatCompletionPrompt, CompletionPrompt
from evals.record import record_sampling
from evals.registry import Registry


def load_embeddings(embeddings_and_text_path: str):
    df = pd.read_csv(embeddings_and_text_path, converters={"embedding": literal_eval})
    assert (
        "text" in df.columns and "embedding" in df.columns
    ), "The embeddings file must have columns named 'text' and 'embedding'"
    return df


def find_top_k_closest_embeddings(
    embedded_prompt: list[float], embs: list[list[float]], k: int
):
    # Normalize the embeddings
    norm_embedded_prompt = embedded_prompt / np.linalg.norm(embedded_prompt)
    norm_embs = embs / np.linalg.norm(embs, axis=1)[:, np.newaxis]

    # Calculate cosine similarity
    cosine_similarities = np.dot(norm_embs, norm_embedded_prompt)

    # Get the indices of the top k closest embeddings
    top_k_indices = np.argsort(cosine_similarities)[-k:]

    return top_k_indices[::-1]


DEFAULT_RETRIEVAL_TEMPLATE = "Use the provided context to answer the question. "


class RetrievalCompletionResult(CompletionResult):
    def __init__(self, response: str) -> None:
        self.response = response

    def get_completions(self) -> list[str]:
        return [self.response.strip()]


class RetrievalCompletionFn(CompletionFn):
    """
    This Completion Function uses embeddings to retrieve the top k relevant docs from a dataset to the prompt, then adds them to the context before calling the completion.
    """

    def __init__(
        self,
        completion_fn: str,
        embeddings_and_text_path: str,
        retrieval_template: str = DEFAULT_RETRIEVAL_TEMPLATE,
        k: int = 4,
        embedding_model: str = "text-embedding-ada-002",
        registry: Optional[Registry] = None,
        registry_path: Optional[str] = None,
        **_kwargs: Any
    ) -> None:
        """
        Args:
            retrieval_template: The template to use for the retrieval. The task prompt will be added to the end of this template.
            k: The number of docs to retrieve from the dataset.
            completion_fn: The completion function to use for the retrieval.
            embeddings_and_text_path: The path to a CSV containing "text" and "embedding" columns.
            registry: Upstream callers may pass in a registry to use.
            registry_path: The path to a registry file to add to default registry.
            _kwargs: Additional arguments to pass to the completion function instantiation.
        """
        registry = Registry() if not registry else registry
        if registry_path:
            registry.add_registry_paths(registry_path)

        self.embeddings_df = load_embeddings(embeddings_and_text_path)

        self.embedding_model = embedding_model
        self.k = k

        self.retrieval_template = retrieval_template
        self.completion_fn_instance = registry.make_completion_fn(completion_fn)

    def __call__(
        self, prompt: Union[str, list[dict]], **kwargs: Any
    ) -> RetrievalCompletionResult:
        """
        Args:
            prompt: The prompt to complete, in either text string or Chat format.
            kwargs: Additional arguments to pass to the completion function call method.
        """
        # Embed the prompt
        embedded_prompt = openai.Embedding.create(
            model=self.embedding_model,
            input=CompletionPrompt(prompt).to_formatted_prompt(),
        )["data"][0]["embedding"]

        embs = self.embeddings_df["embedding"].to_list()

        # Compute the cosine similarity between the prompt and the embeddings
        topk = " ".join(
            self.embeddings_df.iloc[
                find_top_k_closest_embeddings(embedded_prompt, embs, k=self.k)
            ].text.values
        )

        prompt = ChatCompletionPrompt(prompt).to_formatted_prompt()
        retrieval_prompt = [
            {"role": "system", "content": self.retrieval_template + topk}
        ] + prompt

        answer = self.completion_fn_instance(
            prompt=retrieval_prompt, **kwargs
        ).get_completions()[0]
        record_sampling(prompt=retrieval_prompt, sampled=answer)
        return RetrievalCompletionResult(answer)

evals/completion_fns/retrieval.py
GPT-Fathom/GPT-Fathom
"""
This file defines utilities for working with data and files of various types.
"""
import csv
import dataclasses
import gzip
import itertools
import json
import logging
import os
import urllib
from collections.abc import Iterator
from functools import partial
from typing import Any, Sequence, Union

import blobfile as bf
import lz4.frame
import pydantic
import pyzstd

logger = logging.getLogger(__name__)


def gzip_open(filename: str, mode: str = "rb", openhook: Any = open) -> gzip.GzipFile:
    """Wrap the given openhook in gzip."""
    if mode and "b" not in mode:
        mode += "b"

    return gzip.GzipFile(fileobj=openhook(filename, mode), mode=mode)


def lz4_open(
    filename: str, mode: str = "rb", openhook: Any = open
) -> lz4.frame.LZ4FrameFile:
    if mode and "b" not in mode:
        mode += "b"

    return lz4.frame.LZ4FrameFile(openhook(filename, mode), mode=mode)


def zstd_open(filename: str, mode: str = "rb", openhook: Any = open) -> pyzstd.ZstdFile:
    if mode and "b" not in mode:
        mode += "b"

    return pyzstd.ZstdFile(openhook(filename, mode), mode=mode)


def open_by_file_pattern(filename: str, mode: str = "r", **kwargs: Any) -> Any:
    """Can read/write to files on gcs/local with or without gzipping. If file
    is stored on gcs, streams with blobfile. Otherwise use vanilla python open. If
    filename endswith gz, then zip/unzip contents on the fly (note that gcs paths and
    gzip are compatible)"""
    open_fn = partial(bf.BlobFile, **kwargs)
    try:
        if filename.endswith(".gz"):
            return gzip_open(filename, openhook=open_fn, mode=mode)
        elif filename.endswith(".lz4"):
            return lz4_open(filename, openhook=open_fn, mode=mode)
        elif filename.endswith(".zst"):
            return zstd_open(filename, openhook=open_fn, mode=mode)
        else:
            scheme = urllib.parse.urlparse(filename).scheme
            if scheme == "" or scheme == "file":
                return open_fn(
                    os.path.join(
                        os.path.dirname(os.path.abspath(__file__)),
                        "registry",
                        "data",
                        filename,
                    ),
                    mode=mode,
                )
            else:
                return open_fn(filename, mode=mode)
    except Exception as e:
        raise RuntimeError(f"Failed to open: {filename}") from e


def _decode_json(line, path, line_number):
    try:
        return json.loads(line)
    except json.JSONDecodeError as e:
        custom_error_message = f"Error parsing JSON on line {line_number}: {e.msg} at {path}:{line_number}:{e.colno}"
        logger.error(custom_error_message)
        raise ValueError(custom_error_message) from None


def _get_jsonl_file(path):
    logger.info(f"Fetching {path}")
    data = []
    with open_by_file_pattern(path, mode="r") as f:
        return [_decode_json(line, path, i + 1) for i, line in enumerate(f)]


def _get_json_file(path):
    logger.info(f"Fetching {path}")
    with open_by_file_pattern(path, mode="r") as f:
        return json.loads(f.read())


def _stream_jsonl_file(path) -> Iterator:
    logger.info(f"Streaming {path}")
    with bf.BlobFile(path, "r", streaming=True) as f:
        for line in f:
            yield json.loads(line)


def get_lines(path) -> list[dict]:
    """
    Get a list of lines from a file.
    """
    with open_by_file_pattern(path, mode="r") as f:
        return f.readlines()


def get_jsonl(path: str) -> list[dict]:
    """
    Extract json lines from the given path.
    If the path is a directory, look in subpaths recursively.

    Return all lines from all jsonl files as a single list.
    """
    if bf.isdir(path):
        result = []
        for filename in bf.listdir(path):
            if filename.endswith(".jsonl"):
                result += get_jsonl(os.path.join(path, filename))
        return result
    return _get_jsonl_file(path)


def get_jsonls(paths: Sequence[str], line_limit=None) -> list[dict]:
    return list(iter_jsonls(paths, line_limit))


def get_json(path) -> dict:
    if bf.isdir(path):
        raise ValueError("Path is a directory, only files are supported")
    return _get_json_file(path)


def iter_jsonls(paths: Union[str, list[str]], line_limit=None) -> Iterator[dict]:
    """
    For each path in the input, iterate over the jsonl files in that path.
    Look in subdirectories recursively.

    Use an iterator to conserve memory.
    """
    if type(paths) == str:
        paths = [paths]

    def _iter():
        for path in paths:
            if bf.isdir(path):
                for filename in bf.listdir(path):
                    if filename.endswith(".jsonl"):
                        yield from iter_jsonls([os.path.join(path, filename)])
            else:
                yield from _stream_jsonl_file(path)

    return itertools.islice(_iter(), line_limit)


def get_csv(path, fieldnames=None):
    with bf.BlobFile(path, "r", cache_dir="/tmp/bf_cache", streaming=False) as f:
        reader = csv.DictReader(f, fieldnames=fieldnames)
        return [row for row in reader]


def _to_py_types(o: Any) -> Any:
    if isinstance(o, dict):
        return {k: _to_py_types(v) for k, v in o.items()}
    if isinstance(o, list):
        return [_to_py_types(v) for v in o]

    if dataclasses.is_dataclass(o):
        return _to_py_types(dataclasses.asdict(o))

    # pydantic data classes
    if isinstance(o, pydantic.BaseModel):
        return json.loads(o.json())

    return o


class EnhancedJSONEncoder(json.JSONEncoder):
    def default(self, o: Any) -> str:
        return _to_py_types(o)


def jsondumps(o: Any, ensure_ascii: bool = False, **kwargs: Any) -> str:
    return json.dumps(o, cls=EnhancedJSONEncoder, ensure_ascii=ensure_ascii, **kwargs)


def jsondump(o: Any, fp: Any, ensure_ascii: bool = False, **kwargs: Any) -> None:
    json.dump(o, fp, cls=EnhancedJSONEncoder, ensure_ascii=ensure_ascii, **kwargs)


def jsonloads(s: str, **kwargs: Any) -> Any:
    return json.loads(s, **kwargs)


def jsonload(fp: Any, **kwargs: Any) -> Any:
    return json.load(fp, **kwargs)

evals/data.py
GPT-Fathom/GPT-Fathom
import os
from typing import Optional
from urllib.parse import parse_qs, urlparse

import evaluate
from datasets import load_dataset, load_from_disk
from pydantic import BaseModel

import evals
import evals.metrics
from evals.api import CompletionFn
from evals.elsuite.utils import num_tokens_from_messages
from evals.elsuite.utils_execute import compute
from evals.formatting import make_abc
from evals.record import RecorderBase, record_sampling
from evals.registry import n_ctx_from_model_name


CHAT_MODELS = {
    "gpt-3.5",
    "gpt-3.5-frontend",
    "gpt-3.5-turbo",
    "gpt-3.5-turbo-0301",
    "gpt-3.5-turbo-0613",
    "gpt-4",
    "gpt-4-frontend",
    "gpt-4-code-interpreter-frontend",
    "gpt-4-0314",
    "gpt-4-32k",
    "gpt-4-0613",
    "gpt-4-32k-0314",
}

def get_dataset(local, dataset: str):
    if dataset == "humaneval":
        if not local:
            dataset = load_dataset("openai_humaneval", split="test")
        else:
            path = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                "../..",
                "registry",
                "cached_hf_data",
                "humaneval",
                "test",
            )
            dataset = load_from_disk(path)

        return [
            {
                "prompt": sample["prompt"],
                "test": [sample["test"] + f"\ncheck({sample['entry_point']})"],
            }
            for sample in dataset
        ]
    elif dataset == "mbpp":
        if not local:
            dataset = load_dataset("mbpp", "sanitized", split="test")
        else:
            path = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                "../..",
                "registry",
                "cached_hf_data",
                "mbpp",
                "test",
            )
            dataset = load_from_disk(path)

        return [
            {"prompt": sample["prompt"], "test": sample["test_list"]}
            for sample in dataset
        ]


class CODE(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        max_tokens: int = 512,
        dataset: str = None,
        num_samples_per_task: int = 100,
        *args,
        instructions: Optional[str] = "",
        temerpature: float = 0.8,
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert (
            len(completion_fns) == 1
        ), "MultipleChoice only supports one completion fn"
        self.max_tokens = max_tokens
        self.dataset = dataset
        self.num_samples_per_task = num_samples_per_task
        self.temerpature = temerpature
        # The "code_eval" metric executes untrusted model-generated code in Python.
        # Although it is highly unlikely that model-generated code will do something
        # overtly malicious in response to this test suite, model-generated code may act
        # destructively due to a lack of model capability or alignment.
        # Users are strongly encouraged to sandbox this evaluation suite so that it
        # does not perform destructive actions on their host or network. For more
        # information on how OpenAI sandboxes its code, see the paper "Evaluating Large
        # Language Models Trained on Code" (https://arxiv.org/abs/2107.03374).

        # Once you have read this disclaimer and taken appropriate precautions,
        # set the environment variable HF_ALLOW_CODE_EVAL="1". Within Python you can to this
        # with:

        # import os

        # os.environ["HF_ALLOW_CODE_EVAL"] = "1"

        self.OUT_OF_WINDOW = False
        if self.dataset == "humaneval":
            self.k = [1, 10, 100]
        elif self.dataset == "mbpp":
            self.k = [1, 80]

    def make_few_shot_prompts(self, shot):
        path = os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            "../..",
            "registry",
            "cached_hf_data",
            "mbpp",
            "fewshot",
        )
        dataset = load_from_disk(path)
        prompt = ""
        for i in range(shot):
            test_case = "\n".join(dataset[i]["test_list"])
            prompt += "You are an expert Python programmer, and here is your task: "
            prompt += f"{dataset[i]['prompt']}. Your code should pass these tests:\n\n{test_case}\n[BEGIN]\n{dataset[i]['code']}\n[DONE]"
            prompt += "\n\n"

        return prompt

    def pre_process(self, sample):
        if self.dataset == "humaneval":
            prompt = (
                "Complete the code:\n" + sample["prompt"]
                if self.completion_fns[0].model not in CHAT_MODELS
                else "Complete the code and only output the completed code:\n"
                + sample["prompt"]
            )
        elif self.dataset == "mbpp":
            for i in range(3, -1, -1):
                prompt = self.make_few_shot_prompts(i)

                test_case = "\n".join(sample["test"])
                prompt += "You are an expert Python programmer, and here is your task: "
                prompt += f"{sample['prompt']}. Your code should pass these tests:\n\n{test_case}\n[BEGIN]\n"

                if num_tokens_from_messages(
                    [prompt], self.completion_fn.model
                ) + 50 + self.max_tokens <= n_ctx_from_model_name(
                    self.completion_fn.model
                ):
                    break
                self.OUT_OF_WINDOW = True
        return prompt, sample["test"]

    def eval_sample(self, sample, rng):

        prompt, correct_answer = self.pre_process(sample)

        if self.dataset == "humaneval":
            if self.completion_fns[0].model not in CHAT_MODELS:
                preds = [
                    prompt[len("Complete the code:\n") :]
                    + self.completion_fn(
                        prompt=prompt,
                        temperature=self.temerpature,
                        max_tokens=self.max_tokens,
                        stop=["\nclass", "\ndef", "\n#", "\nif"],
                    ).get_completions()[0]
                    + "\n"
                    for _ in range(self.num_samples_per_task)
                ]
            else:
                preds = [
                    self.completion_fn(
                        prompt=prompt,
                        temperature=self.temerpature,
                        max_tokens=self.max_tokens,
                        stop=["\nclass", "\ndef", "\n#", "\nif"],
                    ).get_completions()[0]
                    + "\n"
                    for _ in range(self.num_samples_per_task)
                ]
        elif self.dataset == "mbpp":
            preds = [
                self.completion_fn(
                    prompt=prompt,
                    temperature=self.temerpature,
                    max_tokens=self.max_tokens,
                    stop=["\n[DONE]"],
                ).get_completions()[0]
                + "\n"
                for _ in range(self.num_samples_per_task)
            ]

        # sampled = [[item.split("```")[1][6:] if "```" in item else item for item in preds]]
        sampled = [["a" if None==item else item for item in preds]]
        # print(sampled)
        pass_at_k, results = compute(
            references=correct_answer, predictions=sampled, k=self.k
        )
        if self.dataset == "humaneval":
            evals.record.record_metrics(
                pass_at_1=pass_at_k["pass@1"],
                pass_at_10=pass_at_k["pass@10"],
                pass_at_100=pass_at_k["pass@100"],
            )
        elif self.dataset == "mbpp":
            evals.record.record_metrics(
                pass_at_1=pass_at_k["pass@1"],
                pass_at_80=pass_at_k["pass@80"],
            )

    def eval_sample_batch(self, recorder, samples):
        id = 0
        # for sample in samples:
        data, ideal = [], []
        for i in range(len(samples)):
            prompt, correct_answer = self.pre_process(samples[i])
            data.extend([prompt] * self.num_samples_per_task)
            ideal.append(correct_answer)

        response = self.completion_fn(
            inputs=data,
            temperature=self.temerpature,
            do_sample=True,
            top_p=0.95,
            max_tokens=self.max_tokens,
        )
        for id in range(len(samples)):
            results = response[
                id * self.num_samples_per_task : (id + 1) * self.num_samples_per_task
            ]
            prompt = data[id * self.num_samples_per_task]
            correct_answer = ideal[id]

            stop_sequences = (
                ["\nclass", "\ndef", "\n#", "\nif"]
                if self.dataset == "humaneval"
                else ["\n[DONE]"]
            )
            for i in range(len(results)):
                for x in stop_sequences:
                    if x in results[i]:
                        results[i] = results[i].split(x)[0]

            if self.dataset == "humaneval":
                results = [
                    prompt[len("Complete the code:\n") :] + item for item in results
                ]

            for i in range(len(results)):
                with recorder.as_default_recorder(id):
                    record_sampling(prompt=prompt, sampled=results[i])

            sampled = [["a" if item == None else item for item in results]]
            pass_at_k, results = compute(
                references=correct_answer, predictions=sampled, k=self.k
            )
            with recorder.as_default_recorder(id):
                if self.dataset == "humaneval":
                    evals.record.record_metrics(
                        pass_at_1=pass_at_k["pass@1"],
                        pass_at_10=pass_at_k["pass@10"],
                        pass_at_100=pass_at_k["pass@100"],
                    )
                elif self.dataset == "mbpp":
                    evals.record.record_metrics(
                        pass_at_1=pass_at_k["pass@1"],
                        pass_at_80=pass_at_k["pass@80"],
                    )

    def run(self, recorder: RecorderBase):
        samples = get_dataset(self.local_dataset, self.dataset)
        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)
        import numpy as np

        if self.dataset == "humaneval":
            return {
                "pass@1": np.mean(recorder.get_scores("pass_at_1")),
                "pass@10": np.mean(recorder.get_scores("pass_at_10")),
                "pass@100": np.mean(recorder.get_scores("pass_at_100")),
                "OUT OF WINDOW": self.OUT_OF_WINDOW,
            }
        elif self.dataset == "mbpp":
            return {
                "pass@1": np.mean(recorder.get_scores("pass_at_1")),
                "pass@80": np.mean(recorder.get_scores("pass_at_80")),
                "OUT OF WINDOW": self.OUT_OF_WINDOW,
            }

evals/elsuite/basic/code.py
GPT-Fathom/GPT-Fathom
import numpy as np

import evals
from evals.api import CompletionFn
from evals.elsuite import utils
from evals.prompt.base import chat_prompt_to_text_prompt, is_chat_prompt
from evals.record import RecorderBase, record_sampling


class FuzzyMatch(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        samples_jsonl: str,
        num_few_shot: int = 0,
        few_shot_jsonl: str = None,
        *args,
        max_tokens: int = 100,
        temerpature: float = 0.0,
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert len(completion_fns) == 1, "FuzzyMatch only supports one completion fn"
        self.max_tokens = max_tokens
        self.samples_jsonl = samples_jsonl
        self.num_few_shot = num_few_shot
        self.temerpature = temerpature
        if self.num_few_shot > 0:
            assert (
                few_shot_jsonl is not None
            ), "few shot requires few shot sample dataset"
            self.few_shot_jsonl = few_shot_jsonl
            self.few_shot = evals.get_jsonl(self.few_shot_jsonl)

    def prompt_to_fewshot(self, sample):
        prompt = sample["input"]
        if self.num_few_shot > 0:
            assert is_chat_prompt(sample["input"]), "few shot requires chat prompt"
            prompt = sample["input"][:-1]
            for s in self.few_shot[: self.num_few_shot]:
                prompt += s["sample"]
            prompt += sample["input"][-1:]
        return prompt

    def pre_process(self, test_sample):
        prompt = self.prompt_to_fewshot(test_sample)
        prompt = chat_prompt_to_text_prompt(prompt, Q_A_ENDING=True, new_line_end=False)
        correct_answers = test_sample["ideal"]
        if not isinstance(correct_answers, list):
            correct_answers = [correct_answers]
        return prompt, correct_answers

    def eval_sample(self, test_sample, rng):

        prompt, correct_answers = self.pre_process(test_sample)

        result = self.completion_fn(
            prompt=prompt,
            temperature=self.temerpature,  # Q: why are these hardcoded?
            max_tokens=self.max_tokens,
        )
        sampled = result.get_completions()[0]

        matches = [
            utils.fuzzy_match(sampled, correct_answer)
            for correct_answer in correct_answers
        ]

        evals.record.record_match(
            True in matches,
            expected=correct_answers,
            picked=[sampled for i in range(len(correct_answers)) if matches[i]],
        )
        evals.record.record_metrics(
            accuracy=float(True in matches),
            f1_score=utils.f1_score(sampled, correct_answers),
        )

    def eval_sample_batch(self, recorder, samples):
        data = [self.pre_process(item)[0] for item in samples]
        ideal = [self.pre_process(item)[1] for item in samples]
        results = self.completion_fn(
            inputs=data,
            temperature=self.temerpature,
            max_tokens=50,
        )
        processed_res = [(item) for item in results]
        for i in range(len(processed_res)):
            correct_answers = ideal[i]
            sampled = processed_res[i]
            matches = [
                utils.fuzzy_match(processed_res[i], correct_answer)
                for correct_answer in correct_answers
            ]

            id = str(i)
            with recorder.as_default_recorder(id):
                record_sampling(prompt=data[i], sampled=results[i])
                evals.record.record_match(
                    True in matches,
                    expected=correct_answers,
                    picked=[sampled for i in range(len(correct_answers)) if matches[i]],
                )
                evals.record.record_metrics(
                    accuracy=float(True in matches),
                    f1_score=utils.f1_score(sampled, correct_answers),
                )

    def run(self, recorder: RecorderBase):
        samples = self.get_samples()
        self.eval_all_samples(recorder, samples)

        return {
            "accuracy": np.mean(recorder.get_scores("accuracy")),
            "f1_score": np.mean(recorder.get_scores("f1_score")),
        }

    def run(self, recorder: RecorderBase):
        samples = self.get_samples()
        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)
        return {
            "accuracy": np.mean(recorder.get_scores("accuracy")),
            "f1_score": np.mean(recorder.get_scores("f1_score")),
        }

evals/elsuite/basic/fuzzy_match.py
GPT-Fathom/GPT-Fathom
from typing import Any, Type

from mock import patch
from pytest import mark, raises

from evals.api import DummyCompletionFn
from evals.elsuite.basic.fuzzy_match import FuzzyMatch
from evals.record import DummyRecorder
from evals.utils.test import TestCompletionFn


@mark.parametrize(
    "completion, ideal, expected_metrics",
    [
        ("world", "world", dict(accuracy=1.0, f1_score=1.0)),
        ("world", "foo", dict(accuracy=0, f1_score=0)),
        ("world", ["some foo world", "dummy"], dict(accuracy=1.0, f1_score=0.5)),
    ],
)
def test_eval_sample(
    completion: str,
    ideal: list[str],
    expected_metrics: dict[str, float],
):
    eval = FuzzyMatch(
        completion_fns=[TestCompletionFn(completion)],
        samples_jsonl="",
    )

    recorder = DummyRecorder(None)
    with recorder.as_default_recorder("x"), patch.object(
        recorder, "record_metrics", wraps=recorder.record_metrics
    ) as record_metrics:
        eval.eval_sample(dict(input="Hello", ideal=ideal), None)
        record_metrics.assert_called_once_with(**expected_metrics)


@mark.parametrize(
    "sample, expected_error",
    [
        (None, AssertionError),
        ("", AssertionError),
        (dict(ideal="world"), AssertionError),
        (dict(input="world"), AssertionError),
    ],
)
def test_eval_sample_raises(sample: Any, expected_error: Type):
    eval = FuzzyMatch(
        completion_fns=[DummyCompletionFn()],
        samples_jsonl="",
    )

    with raises(expected_error):
        eval.eval_sample(sample, None)

evals/elsuite/basic/fuzzy_match_test.py
GPT-Fathom/GPT-Fathom
from typing import Any

import evals
import evals.metrics
from evals.api import CompletionFn
from evals.elsuite import utils
from evals.prompt.base import chat_prompt_to_text_prompt, is_chat_prompt
from evals.record import record_sampling

CHAT_MODELS = {
    "gpt-3.5-turbo",
    "gpt-3.5-turbo-0301",
    "gpt-4",
    "gpt-4-0314",
    "gpt-4-32k",
    "gpt-4-32k-0314",
}


class Includes(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        samples_jsonl: str,
        ignore_case: bool = False,
        num_few_shot: int = 0,
        few_shot_jsonl: str = None,
        temerpature: float = 0.0,
        *args,
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert len(completion_fns) == 1, "Includes only supports one completion fn"
        self.samples_jsonl = samples_jsonl
        self.ignore_case = ignore_case
        self.num_few_shot = num_few_shot
        self.temerpature = temerpature
        if self.num_few_shot > 0:
            assert (
                few_shot_jsonl is not None
            ), "few shot requires few shot sample dataset"
            self.few_shot_jsonl = few_shot_jsonl
            self.few_shot = evals.get_jsonl(self.few_shot_jsonl)

    def prompt_to_fewshot(self, sample):
        prompt = sample["input"]
        if self.num_few_shot > 0:
            assert is_chat_prompt(sample["input"]), "few shot requires chat prompt"
            prompt = sample["input"][:-1]
            for s in self.few_shot[: self.num_few_shot]:
                prompt += s["sample"]
            prompt += sample["input"][-1:]
        return prompt

    def pre_process(self, test_sample):
        prompt = self.prompt_to_fewshot(test_sample)
        if self.completion_fns[0].model not in CHAT_MODELS:
            if self.num_few_shot == 0:
                prompt = chat_prompt_to_text_prompt(
                    prompt, Q_A_ENDING=True, new_line_end=False
                )
            else:
                prompt = chat_prompt_to_text_prompt(
                    prompt, Q_A_ENDING=True, new_line_end=False
                )

        correct_answers = test_sample["ideal"]
        if not isinstance(correct_answers, list):
            correct_answers = [correct_answers]
        return prompt, correct_answers

    def eval_sample(self, sample: Any, *_):

        prompt, ideal = self.pre_process(sample)
        result = self.completion_fn(
            prompt=prompt,
            temperature = self.temerpature
        )
        sampled = result.get_completions()[0]

        includes_answer = any(
            [
                utils.get_answer(sampled, ref, self.ignore_case) is not None
                for ref in ideal
            ]
        )
        evals.record.record_match(
            includes_answer, expected=sample["ideal"], picked=sampled, sampled=sampled
        )
        return includes_answer

    def eval_sample_batch(self, recorder, samples):
        data = [self.pre_process(item)[0] for item in samples]
        ideal = [self.pre_process(item)[1] for item in samples]
        results = self.completion_fn(inputs=data, temperature=self.temerpature, max_tokens=50)
        processed_res = [(item) for item in results]
        for i in range(len(processed_res)):
            correct_answers = ideal[i]
            sampled = processed_res[i]
            includes_answer = any(
                [
                    utils.get_answer(sampled, ref, self.ignore_case) is not None
                    for ref in correct_answers
                ]
            )

            id = str(i)
            with recorder.as_default_recorder(id):
                record_sampling(prompt=data[i], sampled=results[i])
                evals.record.record_match(
                    includes_answer,
                    expected=correct_answers,
                    picked=sampled,
                    sampled=sampled,
                )

    def run(self, recorder):
        samples = self.get_samples()
        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)
        events = recorder.get_events("match")
        return {
            "accuracy": evals.metrics.get_accuracy(events),
        }

evals/elsuite/basic/includes.py
GPT-Fathom/GPT-Fathom
from typing import Any, Type, Union

from mock import patch
from pytest import mark, raises

from evals.api import DummyCompletionFn
from evals.elsuite.basic.includes import Includes
from evals.record import DummyRecorder
from evals.utils.test import TestCompletionFn


@mark.parametrize(
    "completion, ideal, expected_match, ignore_case",
    [
        ("world", "world", True, False),
        ("world", "wOrLd", True, True),
        ("world", ["world"], True, False),
        ("world", ["foo", "bar"], False, False),
        ("world", ["worldfoo", "worldbar"], False, False),
    ],
)
def test_eval_sample(
    completion: str,
    ideal: Union[str, list[str]],
    expected_match: bool,
    ignore_case: bool,
):
    eval = Includes(
        completion_fns=[TestCompletionFn(completion)],
        samples_jsonl="",
        ignore_case=ignore_case,
    )

    recorder = DummyRecorder(None)
    with recorder.as_default_recorder("x"), patch.object(
        recorder, "record_match", wraps=recorder.record_match
    ) as record_match:
        eval.eval_sample(dict(input="Hello", ideal=ideal), None)
        record_match.assert_called_once_with(
            expected_match, expected=ideal, picked=completion, sampled=completion
        )


@mark.parametrize(
    "sample, expected_error",
    [
        (None, AssertionError),
        ("", AssertionError),
        (dict(ideal="world"), AssertionError),
        (dict(input="world"), AssertionError),
    ],
)
def test_eval_sample_raises(sample: Any, expected_error: Type):
    eval = Includes(
        completion_fns=[DummyCompletionFn()],
        samples_jsonl="",
    )

    with raises(expected_error):
        eval.eval_sample(sample, None)

evals/elsuite/basic/includes_test.py
GPT-Fathom/GPT-Fathom
import re
from typing import Optional
from urllib.parse import parse_qs, urlparse

from datasets import load_dataset

import evals
import evals.metrics
from evals.api import CompletionFn
from evals.elsuite.utils import num_tokens_from_messages
from evals.formatting import make_abc
from evals.record import RecorderBase, record_sampling
from evals.registry import n_ctx_from_model_name
from evals.utils.math_util import (
    clean_numbers,
    is_equiv,
    last_boxed_only,
    last_boxed_only_string,
    remove_boxed,
)


def check_match(sampled, ans, dataset):
    if dataset == "mgsm":
        ANS_RE = re.compile(r"(-?\d[\d,]*\.?\d*)(?![\d-])")
        matches = ANS_RE.findall(sampled)
        if matches:
            match_str = matches[-1]
            return match_str, ans
        else:
            return None, ans
    if dataset == "gsm8k":
        ANS_RE = re.compile(r"\d+")
        match = re.findall(r"\d+", sampled)
        reference_answer = ans.split("####")[1].strip()
        ### LOOSE
        if "The answer is " not in sampled:
            match = re.findall(r"\d+", sampled)
            match_pred = re.findall(r"\d+", reference_answer)
            if match:
                match_str = match[-1]
                return match_str, match_pred
        else:
            extracted = sampled.split("The answer is ")[1]
            match = re.findall(r"\d+", extracted)
            match_pred = re.findall(r"\d+", reference_answer)
            if match:
                match_str = match[0]
                return match_str, match_pred
        ## STRICT PROCESSING
        # if "The answer is " in sampled:
        #     extracted = sampled.split("The answer is ")[1]
        #     match = re.findall(r'\d+', extracted)
        #     match_pred = re.findall(r'\d+', reference_answer)
        #     if match:
        #         match_str = match[0]
        #         return match_str, match_pred
        # if match:
        # match_str = match[-1]
        return None, reference_answer
        # else:
        #     return None, reference_answer
        # return last_number
    if dataset == "math":

        sampled = last_boxed_only_string(sampled)
        sampled = remove_boxed(sampled)

        if is_equiv(sampled, ans):
            return ans, ans
        else:
            return sampled, ans


class MATH_PROBLEM(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        num_few_shot: int = 0,
        few_shot: str = None,
        samples_jsonl: str = None,
        few_shot_jsonl: str = None,
        max_tokens: int = 256,
        *args,
        instructions: Optional[str] = "",
        temerpature: float = 0.0,
        example_q_alias: str = "Q: ",
        example_a_alias: str = "A: ",
        q_alias: str = "Q: ",
        a_alias: str = "A: ",
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert (
            len(completion_fns) == 1
        ), "MultipleChoice only supports one completion fn"
        self.instructions = "Follow the given examples and answer the question."
        self.few_shot = num_few_shot
        self.few_shot_prompt = ""
        self.max_tokens = max_tokens
        self.sample_jsonl = samples_jsonl
        self.temerpature = temerpature
        self.prompt_template = [example_q_alias, example_a_alias, q_alias, a_alias]
        self.OUT_OF_WINDOW = False
        if self.few_shot > 0:
            self.example = evals.get_jsonl(few_shot_jsonl)

    def make_few_shot_prompt(self, num_shot):

        few_shot_prompt = ""
        for i in range((num_shot)):
            few_shot_prompt += self.prompt_template[0] + self.example[i]["question"] + "\n"
            few_shot_prompt += self.prompt_template[1] + self.example[i]["answer"] + "\n\n"

        return few_shot_prompt

    def pre_process(self, sample):
        for i in range(self.few_shot, -1, -1):
            few_shot_prompt = self.make_few_shot_prompt(i)
            prompt = (
                self.instructions
                + "\n\n"
                + few_shot_prompt
                + "\n\n"
                + self.prompt_template[2]
                + sample["question"]
                + f"\n{self.prompt_template[3]}Let's think step by step"
            )
            if num_tokens_from_messages(
                [prompt], self.completion_fn.model
            ) + 50 + self.max_tokens <= n_ctx_from_model_name(self.completion_fn.model):
                break
            self.OUT_OF_WINDOW = True

        return prompt, sample["answer"]

    def post_process(self, sampled, correct_answer):
        detected = False
        extracted, correct_answer = check_match(
            sampled, correct_answer, self.sample_jsonl.split("/")[0]
        )
        if extracted != None:
            sampled = extracted
            detected = True
        return sampled, correct_answer, detected

    def eval_sample(self, sample, rng):
        prompt, correct_answer = self.pre_process(sample)

        result = self.completion_fn(
            prompt=prompt,
            temperature=self.temerpature,
            max_tokens=self.max_tokens,
        )
        sampled = result.get_completions()[0]
        sampled, correct_answer, detected = self.post_process(sampled, correct_answer)
        evals.record_and_check_match(
            prompt=prompt, sampled=sampled, expected=correct_answer, detected=detected
        )

    def eval_sample_batch(self, recorder, samples):
        data = [self.pre_process(item)[0] for item in samples]
        ideal = [self.pre_process(item)[1] for item in samples]
        results = self.completion_fn(
            inputs=data,
            temperature=self.temerpature,
            max_tokens=self.max_tokens,
        )
        processed_res, detected, correct_answers = [], [], []
        for i in range(len(results)):
            a, answer, b = self.post_process(results[i], ideal[i])
            processed_res.append(a)
            detected.append(b)
            correct_answers.append(answer)

        # processed_res = [self.post_process(item)[0] for item in results]
        # detected = [self.post_process(item)[1] for item in results]
        for i in range(len(processed_res)):
            id = str(i)
            with recorder.as_default_recorder(id):
                record_sampling(prompt=data[i], sampled=results[i])
                evals.record_and_check_match(
                    prompt=data[i],
                    sampled=processed_res[i],
                    detected=detected[i],
                    expected=correct_answers[i],
                )

    def run(self, recorder: RecorderBase):
        samples = evals.get_jsonl(self.sample_jsonl)

        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)
        return {
            "accuracy": evals.metrics.get_accuracy(recorder.get_events("match")),
            "detected": evals.metrics.get_picked(recorder.get_events("match")),
            "OUT OF WINDOW": self.OUT_OF_WINDOW,
        }

evals/elsuite/basic/math_problem.py
GPT-Fathom/GPT-Fathom
import os
import random
from typing import Optional
from urllib.parse import parse_qs, urlparse

from datasets import load_dataset, load_from_disk
from pydantic import BaseModel

import evals
import evals.metrics
from evals.api import CompletionFn
from evals.elsuite.dataset_specific.lambada import CHAT_MODELS
from evals.elsuite.utils import num_tokens_from_messages
from evals.formatting import make_abc
from evals.record import RecorderBase, record_sampling
from evals.registry import n_ctx_from_model_name


class Sample(BaseModel):
    question: str
    answers: list[str]
    label: int


def get_dataset(local, url: str) -> list[Sample]:
    parsed = urlparse(url)
    if parsed.scheme == "hf":
        query = parse_qs(parsed.query)
        query = {k: v[0] for k, v in query.items()}

        path = parsed.netloc
        if not local:
            dataset = load_dataset(path, **query)
        if path == "hellaswag":
            if local:
                path = os.path.join(
                    os.path.dirname(os.path.abspath(__file__)),
                    "../..",
                    "registry",
                    "cached_hf_data",
                    "hellaswag",
                    "test" if query["split"] == "validation" else "fewshot",
                )
                dataset = load_from_disk(path)
            return [
                Sample(
                    question=sample["ctx"],
                    answers=sample["endings"],
                    label=int(sample["label"]),
                )
                for sample in dataset
            ]
        elif path == "hendrycks_test":
            if local:
                print(query["split"])
                path = os.path.join(
                    os.path.dirname(os.path.abspath(__file__)),
                    "../..",
                    "registry",
                    "cached_hf_data",
                    "mmlu",
                    query["name"],
                    "test" if query["split"] == "test" else "fewshot",
                )
                dataset = load_from_disk(path)
            return [
                Sample(
                    question=sample["question"],
                    answers=sample["choices"],
                    label=sample["answer"],
                )
                for sample in dataset
            ]
        elif path == "winogrande":
            if local:
                path = os.path.join(
                    os.path.dirname(os.path.abspath(__file__)),
                    "../..",
                    "registry",
                    "cached_hf_data",
                    "winogrande",
                    "test" if query["split"] == "validation" else "fewshot",
                )
                dataset = load_from_disk(path)
            return [
                Sample(
                    question=sample["sentence"],
                    answers=[sample["option1"], sample["option2"]],
                    label=int(sample["answer"]) - 1,
                )
                for sample in dataset
            ]
    raise ValueError(f"Unknown question dataset {url}")


def load_from_file(file: str) -> list[Sample]:
    data = evals.get_jsonl(file)
    return [
        Sample(
            question=sample["question"],
            answers=sample["choices"],
            label=sample["answer"],
        )
        for sample in data
    ]


class MultipleChoice(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        dataset: str = None,
        num_few_shot: int = 0,
        few_shot: str = None,
        sample_jsonl: str = None,
        few_shot_jsonl: str = None,
        no_MC_prompt: bool = False,
        CoT: bool = False,
        max_tokens: int = 10,
        temerpature: float = 0.0,
        *args,
        instructions: Optional[str] = "",
        example_q_alias: str = "Q: ",
        example_a_alias: str = "A: ",
        q_alias: str = "Q: ",
        a_alias: str = "A: ",
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert (
            len(completion_fns) == 1
        ), "MultipleChoice only supports one completion fn"
        self.dataset = dataset
        self.instructions = instructions
        self.few_shot = num_few_shot
        self.few_shot_prompt = ""
        self.sample_jsonl = sample_jsonl
        self.no_MC_prompt = no_MC_prompt
        self.temerpature = temerpature
        self.max_tokens = max_tokens
        self.prompt_template = [example_q_alias, example_a_alias, q_alias, a_alias]
        self.CoT = CoT
        if self.few_shot > 0:
            if few_shot_jsonl == None:
                self.example = get_dataset(self.local_dataset, few_shot)
            else:
                self.example = load_from_file(few_shot_jsonl)
        self.OUT_OF_WINDOW = False

    def make_few_shot_prompt(self, num_shot):
        few_shot_prompt = ""
        rng = random.Random(47)
        for i in range((num_shot)):
            options, correct_answer, ans_ctx = make_abc(
                answers=self.example[i].answers,
                correct_idx=self.example[i].label,
                rng=rng,
            )
            few_shot_prompt += self.prompt_template[0] + self.example[i].question + "\n\n" + options  #
            if not self.no_MC_prompt:
                few_shot_prompt += (
                    "\nPlease answer with the letter of the correct answer.\n"
                )
            else:
                few_shot_prompt += "\n"
            few_shot_prompt += self.prompt_template[1] + ans_ctx + "\n\n"

        return few_shot_prompt

    def pre_process(self, sample):
        options, correct_answer, _ = make_abc(
            answers=sample.answers,
            correct_idx=sample.label,
            rng=random.Random(47),
        )
        for i in range(self.few_shot, -1, -1):
            few_shot_prompt = self.make_few_shot_prompt(i)
            prompt = (
                self.instructions
                + "\n\n"
                + few_shot_prompt
                + self.prompt_template[2]
                + sample.question
                + "\n\n"
                + options
            )
            if not self.no_MC_prompt:
                prompt += f"\nPlease answer with the letter of the correct answer.\n{self.prompt_template[3]}"
            else:
                prompt += f"\n{self.prompt_template[3]}"
            if self.CoT:
                prompt += "Let's think step by step. "

            if num_tokens_from_messages(
                [prompt], self.completion_fn.model
            ) + 50 + self.max_tokens <= n_ctx_from_model_name(self.completion_fn.model):
                break
            self.OUT_OF_WINDOW = True
        return prompt, correct_answer

    def post_process(self, sampled):
        options = ["(" + chr(ord("A") + idx) + ")" for idx in range(26)]
        ans = sampled
        for option in options:
            if option in sampled:
                ans = option[1]
                break
        if ans == sampled:
            sampled = sampled.lstrip()
            ans = sampled
        sampled = ans
        return sampled

    def eval_sample(self, sample, rng):

        prompt, correct_answer = self.pre_process(sample)

        result = self.completion_fn(
            prompt=prompt,
            temperature=self.temerpature,
            max_tokens=self.max_tokens,
        )
        sampled = result.get_completions()[0]
        evals.record_and_check_match(
            prompt=prompt,
            sampled=self.post_process(sampled),
            expected=correct_answer,
        )

    def eval_sample_batch(self, recorder, samples):
        data = [self.pre_process(item)[0] for item in samples]
        ideal = [self.pre_process(item)[1] for item in samples]
        results = self.completion_fn(
            inputs=data,
            temperature=self.temerpature,
            max_tokens=self.max_tokens,
        )
        processed_res = [self.post_process(item) for item in results]
        for i in range(len(processed_res)):
            id = str(i)
            with recorder.as_default_recorder(id):
                record_sampling(prompt=data[i], sampled=results[i])
                evals.record_and_check_match(
                    prompt=data[i],
                    sampled=processed_res[i],
                    expected=ideal[i],
                )

    def run(self, recorder: RecorderBase):
        if self.sample_jsonl == None:
            samples = get_dataset(self.local_dataset, self.dataset)
        else:
            samples = load_from_file(self.sample_jsonl)

        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)

        return {
            "accuracy": evals.metrics.get_accuracy(recorder.get_events("match")),
            "OUT OF WINDOW": self.OUT_OF_WINDOW,
        }

evals/elsuite/basic/multiple_choice.py
GPT-Fathom/GPT-Fathom
import random
import re
from typing import Optional
from urllib.parse import parse_qs, urlparse

import numpy as np
from datasets import load_dataset

import evals
import evals.metrics
from evals.api import CompletionFn
from evals.elsuite import utils
from evals.elsuite.utils import num_tokens_from_messages
from evals.formatting import make_abc
from evals.record import RecorderBase, record_sampling
from evals.registry import n_ctx_from_model_name


class READ_COMPRE(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        num_few_shot: int = 0,
        samples_jsonl: str = None,
        few_shot_jsonl: str = None,
        max_tokens: int = 30,
        MC: bool = False,
        no_MC_prompt: bool = False,
        *args,
        instructions: Optional[str] = "",
        temerpature: float = 0.0,
        example_q_alias: str = "Q: ",
        example_a_alias: str = "A: ",
        q_alias: str = "Q: ",
        a_alias: str = "A: ",
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert (
            len(completion_fns) == 1
        ), "MultipleChoice only supports one completion fn"
        self.instructions = (
            "The following are questions (with answers) about reading comprehension. "
        )
        self.few_shot = num_few_shot
        self.few_shot_prompt = ""
        self.max_tokens = max_tokens
        self.sample_jsonl = samples_jsonl
        self.MC = MC
        self.OUT_OF_WINDOW = False
        self.prompt_template = [example_q_alias, example_a_alias, q_alias, a_alias]
        self.temerpature = temerpature
        if self.few_shot > 0:
            self.example = evals.get_jsonl(few_shot_jsonl)
        self.no_MC_prompt = no_MC_prompt

    def make_few_shot_prompt(self, num_shot):
        few_shot_prompt = ""
        if not self.MC:
            for i in range((num_shot)):
                few_shot_prompt += "Passage: " + self.example[i]["passage"] + "\n"
                few_shot_prompt += "Question: " + self.example[i]["question"] + "\n"
                few_shot_prompt += "Answer: " + self.example[i]["ideal"][0] + "\n\n"
        else:
            rng = random.Random(47)
            for i in range((num_shot)):
                options, correct_answer, ans_ctx = make_abc(
                    answers=self.example[i]["options"],
                    correct_idx=self.example[i]["ideal"],
                    rng=rng,
                )
                few_shot_prompt += "Passage: " + self.example[i]["passage"] + "\n"
                few_shot_prompt += self.prompt_template[0] + self.example[i]["question"] + "\n" + options

                if not self.no_MC_prompt:
                    few_shot_prompt += (
                        "\nPlease answer with the letter of the correct answer.\n"
                    )
                else:
                    few_shot_prompt += "\n"
                few_shot_prompt += self.prompt_template[1] + ans_ctx + "\n\n"

        return few_shot_prompt

    def pre_process(self, sample):
        if not self.MC:
            correct_answers = sample["ideal"]
            if not isinstance(correct_answers, list):
                correct_answers = [correct_answers]
            prompt = (
                self.instructions
                + "\n\n"
                + self.make_few_shot_prompt(self.few_shot)
                + "\n\n"
            )
            prompt += (
                "Passage: "
                + sample["passage"]
                + "\n"
                + "Question: "
                + sample["question"]
                + "\nAnswer: "
            )
            return prompt, correct_answers
        else:
            options, correct_answer, _ = make_abc(
                answers=sample["options"],
                correct_idx=sample["ideal"],
                rng=random.Random(47),
            )
            for i in range(self.few_shot, -1, -1):
                few_shot_prompt = self.make_few_shot_prompt(i)
                prompt = self.instructions + "\n\n" + few_shot_prompt + "\n\n"
                prompt += (
                    "Passage: "
                    + sample["passage"]
                    + "\n"
                    + "Q: "
                    + sample["question"]
                    + options
                )

                if not self.no_MC_prompt:
                    prompt += (
                        "\nPlease answer with the letter of the correct answer.\nA: "
                    )
                else:
                    prompt += "\nA: "
                if num_tokens_from_messages(
                    [prompt], self.completion_fn.model
                ) + 50 + self.max_tokens <= n_ctx_from_model_name(
                    self.completion_fn.model
                ):
                    break
                self.OUT_OF_WINDOW = True
            return prompt, correct_answer

    def eval_sample(self, sample, rng):

        if not self.MC:
            prompt, correct_answers = self.pre_process(sample)
            result = self.completion_fn(
                prompt=prompt,
                temperature=self.temerpature,
                max_tokens=self.max_tokens,
            )
            sampled = result.get_completions()[0]

            matches = [
                utils.fuzzy_match(sampled, correct_answer)
                for correct_answer in correct_answers
            ]

            evals.record.record_match(
                True in matches,
                expected=correct_answers,
                picked=[sampled for i in range(len(correct_answers)) if matches[i]],
            )
            evals.record.record_metrics(
                accuracy=float(True in matches),
                f1_score=utils.f1_score(sampled, correct_answers),
            )
        else:
            prompt, correct_answer = self.pre_process(sample)
            result = self.completion_fn(
                prompt=prompt,
                temperature=self.temerpature,
                max_tokens=50,
            )
            sampled = result.get_completions()[0]
            options = ["(" + chr(ord("A") + idx) + ")" for idx in range(26)]
            ans = sampled
            for option in options:
                if option in sampled:
                    ans = option[1]
                    break
            sampled = ans
            evals.record_and_check_match(
                prompt=prompt,
                sampled=sampled,
                expected=correct_answer,
            )

    def eval_sample_batch(self, recorder, samples):
        data = [self.pre_process(item)[0] for item in samples]
        ideal = [self.pre_process(item)[1] for item in samples]
        results = self.completion_fn(
            inputs=data,
            temperature=self.temerpature,
            max_tokens=30,
        )
        processed_res = [(item) for item in results]
        for i in range(len(processed_res)):
            id = str(i)
            with recorder.as_default_recorder(id):
                if self.MC:
                    record_sampling(prompt=data[i], sampled=results[i])
                    options = [chr(ord("A") + idx) + ")" for idx in range(26)]
                    sampled, correct_answer = results[i], ideal[i]
                    ans = sampled
                    for option in options:
                        if option in sampled:
                            ans = option[0]
                            break
                    sampled = ans
                    evals.record_and_check_match(
                        prompt=data[i],
                        sampled=sampled,
                        expected=correct_answer,
                    )
                else:
                    record_sampling(prompt=data[i], sampled=results[i])
                    sampled, correct_answers = results[i], ideal[i]

                    matches = [
                        utils.fuzzy_match(sampled, correct_answer)
                        for correct_answer in correct_answers
                    ]

                    evals.record.record_match(
                        True in matches,
                        expected=correct_answers,
                        picked=[
                            sampled for i in range(len(correct_answers)) if matches[i]
                        ],
                    )
                    evals.record.record_metrics(
                        accuracy=float(True in matches),
                        f1_score=utils.f1_score(sampled, correct_answers),
                    )

    def run(self, recorder: RecorderBase):
        samples = evals.get_jsonl(self.sample_jsonl)

        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)
        print("OUT OF WINDOW: ", self.OUT_OF_WINDOW)
        return {
            "accuracy": np.mean(recorder.get_scores("accuracy"))
            if not self.MC
            else evals.metrics.get_accuracy(recorder.get_events("match")),
            "f1_score": np.mean(recorder.get_scores("f1_score")),
            "OUT OF WINDOW": self.OUT_OF_WINDOW,
        }

evals/elsuite/basic/reading_comprehension.py
GPT-Fathom/GPT-Fathom
import json
import os
import re
from typing import Optional
from urllib.parse import parse_qs, urlparse

import pandas as pd
from datasets import load_dataset

import evals
import evals.metrics
from evals.api import CompletionFn
from evals.elsuite.utils import num_tokens_from_messages
from evals.formatting import make_abc
from evals.record import RecorderBase, record_sampling
from evals.registry import n_ctx_from_model_name
from evals.utils.math_util import (
    clean_numbers,
    is_equiv,
    last_boxed_only,
    last_boxed_only_string,
    remove_boxed,
)


class AGIEVAL(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        num_few_shot: int = 0,
        task: str = None,
        sample_jsonl: str = None,
        few_shot_jsonl: str = None,
        max_tokens: int = 512,
        type: str = None,
        CoT: bool = False,
        Cn: bool = False,
        *args,
        instructions: Optional[str] = "",
        temerpature: float = 0.0,
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert (
            len(completion_fns) == 1
        ), "MultipleChoice only supports one completion fn"
        self.few_shot = num_few_shot
        self.max_tokens = max_tokens
        self.sample_jsonl = sample_jsonl
        self.OUT_OF_WINDOW = False
        self.type = type
        self.cn = Cn
        self.CoT = CoT
        self.temerpature = temerpature
        self.few_shot_samples = []
        if self.type == "MC":
            self.instructions = (
                "Follow the given samples and answer the following multiple choice question."
                if not self.cn
                else "回答下列选择题"
            )
        elif self.type == "IMC":
            self.instructions = (
                "Follow the given samples and answer the following multiple select question."
                if not self.cn
                else "回答下列多选题"
            )
        else:
            self.instructions = (
                "Follow the given samples and answer the following cloze question."
                if not self.cn
                else "回答下列填空题"
            )
        if self.few_shot > 0:
            path = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                "../..",
                "registry",
                "data",
                few_shot_jsonl,
            )
            data = pd.read_csv(path)
            context_row = [0, 1, 3, 5, 7, 9]
            explanation_row = [0, 2, 4, 6, 8, 10]
            raw_prompts_context = pd.read_csv(
                path,
                header=0,
                skiprows=lambda x: x not in context_row,
                keep_default_na=False,
            )
            raw_prompts_explanation = pd.read_csv(
                path,
                header=0,
                skiprows=lambda x: x not in explanation_row,
                keep_default_na=False,
            ).replace(r"\n\n", "\n", regex=True)
            contexts = []
            import ast

            if task == "sat-en-without-passage":
                task = "sat-en"
            for line in list(raw_prompts_context[task]):
                if line:
                    # print(line)
                    contexts.append(ast.literal_eval(line))
            explanations = [exp for exp in raw_prompts_explanation[task] if exp]
            self.few_shot = len(explanations)
            for i in range(len(explanations)):
                sample = contexts[i]
                self.few_shot_samples.append(
                    {
                        "passage": sample.get("passage", ""),
                        "question": sample.get("question", ""),
                        "answer": sample.get("answer", ""),
                        "options": sample.get("options", ""),
                        "label": sample.get("label", ""),
                        "explanation": explanations[i],
                    }
                )
            self.few_shot_samples = self.few_shot_samples[: self.few_shot]

    def make_few_shot_prompt(self, num_shot):

        few_shot_prompt = ""
        if not self.cn:
            if self.type == "MC":

                for i in range((num_shot)):
                    if self.few_shot_samples[i]["passage"] not in [None, ""]:
                        few_shot_prompt += (
                            "Passage: " + self.few_shot_samples[i]["passage"] + "\n"
                        )
                    few_shot_prompt += (
                        "Q: " + self.few_shot_samples[i]["question"] + "\n"
                    )
                    for item in self.few_shot_samples[i]["options"]:
                        few_shot_prompt += item + "\n"
                    label = self.few_shot_samples[i]["label"]
                    if self.CoT:
                        few_shot_prompt += (
                            "A: Let's think step by step. "
                            + self.few_shot_samples[i]["explanation"]
                            + "\n"
                        )
                        few_shot_prompt += "So the answer is (" + label + ") "
                        few_shot_prompt += (
                            self.few_shot_samples[i]["options"][ord(label) - ord("A")]
                            + "\n\n"
                        )
                    else:
                        few_shot_prompt += (
                            "A: The answer is ("
                            + label
                            + ") "
                            + self.few_shot_samples[i]["options"][ord(label) - ord("A")]
                            + "\n\n"
                        )

            elif self.type == "Cloze":
                for i in range((num_shot)):
                    if self.few_shot_samples[i]["passage"] not in [None, ""]:
                        few_shot_prompt += (
                            "Passage: " + self.few_shot_samples[i]["passage"] + "\n"
                        )
                    few_shot_prompt += (
                        "Q: " + self.few_shot_samples[i]["question"] + "\n"
                    )
                    if self.CoT:
                        few_shot_prompt += (
                            "A: Let's think step by step. "
                            + self.few_shot_samples[i]["explanation"]
                            + "\n"
                        )
                        few_shot_prompt += (
                            "So the answer is "
                            + self.few_shot_samples[i]["answer"]
                            + "\n\n"
                        )
                    else:
                        few_shot_prompt += (
                            "A: " + self.few_shot_samples[i]["answer"] + "\n\n"
                        )
        else:

            if self.type == "MC":
                for i in range((num_shot)):
                    if self.few_shot_samples[i]["passage"] not in [None, ""]:
                        few_shot_prompt += (
                            "文章: " + self.few_shot_samples[i]["passage"] + "\n"
                        )
                    few_shot_prompt += (
                        "问题: " + self.few_shot_samples[i]["question"] + "\n"
                    )
                    for item in self.few_shot_samples[i]["options"]:
                        few_shot_prompt += item + "\n"
                    label = self.few_shot_samples[i]["label"]
                    if self.CoT:
                        few_shot_prompt += (
                            "回答: 让我们一步步思考。 "
                            + self.few_shot_samples[i]["explanation"]
                            + "\n"
                        )
                        few_shot_prompt += "所以答案是" + "(" + label + ") "
                        few_shot_prompt += (
                            self.few_shot_samples[i]["options"][ord(label) - ord("A")]
                            + "\n\n"
                        )
                    else:
                        few_shot_prompt += (
                            "回答：("
                            + label
                            + ") "
                            + self.few_shot_samples[i]["options"][ord(label) - ord("A")]
                            + "\n\n"
                        )

            elif self.type == "IMC":
                for i in range((num_shot)):
                    if self.few_shot_samples[i]["passage"] not in [None, ""]:
                        few_shot_prompt += (
                            "文章: " + self.few_shot_samples[i]["passage"] + "\n"
                        )
                    few_shot_prompt += (
                        "问题: " + self.few_shot_samples[i]["question"] + "\n"
                    )
                    for item in self.few_shot_samples[i]["options"]:
                        few_shot_prompt += item + "\n"
                    if isinstance(self.few_shot_samples[i]["label"], str):
                        self.few_shot_samples[i]["label"] = [
                            self.few_shot_samples[i]["label"]
                        ]

                    if self.CoT:
                        few_shot_prompt += (
                            "回答: 让我们一步步思考。"
                            + self.few_shot_samples[i]["explanation"]
                            + "\n"
                        )
                        few_shot_prompt += "所以答案是\n"
                        for item in self.few_shot_samples[i]["label"]:
                            few_shot_prompt += (
                                f"({item}) "
                                + self.few_shot_samples[i]["options"][
                                    ord(item) - ord("A")
                                ]
                                + "\n"
                            )
                        few_shot_prompt += "\n"
                    else:
                        few_shot_prompt += "回答："
                        for item in self.few_shot_samples[i]["label"]:
                            few_shot_prompt += (
                                f"({item}) "
                                + self.few_shot_samples[i]["options"][
                                    ord(item) - ord("A")
                                ]
                                + "\n"
                            )
                        few_shot_prompt += "\n"

            elif self.type == "Cloze":
                for i in range((num_shot)):
                    if self.few_shot_samples[i]["passage"] not in [None, ""]:
                        few_shot_prompt += (
                            "文章: " + self.few_shot_samples[i]["passage"] + "\n"
                        )
                    few_shot_prompt += (
                        "问题: " + self.few_shot_samples[i]["question"] + "\n"
                    )
                    if self.CoT:
                        few_shot_prompt += (
                            "回答: 让我们一步步思考。"
                            + self.few_shot_samples[i]["explanation"]
                            + "\n"
                        )
                        few_shot_prompt += (
                            "所以答案是" + self.few_shot_samples[i]["answer"] + "\n\n"
                        )
                    else:
                        few_shot_prompt += (
                            "回答：" + self.few_shot_samples[i]["answer"] + "\n\n"
                        )
        return few_shot_prompt

    def pre_process(self, sample):
        for i in range(self.few_shot, -1, -1):
            few_shot_prompt = self.make_few_shot_prompt(i)
            prompt = self.instructions + "\n\n" + few_shot_prompt + "\n\n"

            ##### Process data pair into text

            if not self.cn:
                if self.type == "MC":
                    if sample["passage"] not in [None, ""]:
                        prompt += "Passage: " + sample["passage"] + "\n"
                    prompt += "Q: " + sample["question"] + "\n"
                    for item in sample["options"]:
                        prompt += item + "\n"
                    if self.CoT:
                        prompt += "A: Let's think step by step. "
                    else:
                        prompt += "A: "

                elif self.type == "IMC":
                    if sample["passage"] not in [None, ""]:
                        prompt += "Passage: " + sample["passage"] + "\n"
                    prompt += "Q: " + sample["question"] + "\n"
                    for item in sample["options"]:
                        prompt += item + "\n"
                    if isinstance(sample["label"], str):
                        sample["label"] = [sample["label"]]

                    if self.CoT:
                        prompt += "A: Let's think step by step. "
                    else:
                        prompt += "A: "

                elif self.type == "Cloze":
                    if sample["passage"] not in [None, ""]:
                        prompt += "Passage: " + sample["passage"] + "\n"
                    prompt += "Q: " + sample["question"] + "\n"
                    if self.CoT:
                        prompt += "A: Let's think step by step. "
                    else:
                        prompt += "A: "
            else:

                if self.type == "MC":
                    if sample["passage"] not in [None, ""]:
                        prompt += "文章: " + sample["passage"] + "\n"
                    prompt += "问题: " + sample["question"] + "\n"
                    for item in sample["options"]:
                        prompt += item + "\n"
                    if self.CoT:
                        prompt += "回答: 让我们一步步思考。 "
                    else:
                        prompt += "回答："

                elif self.type == "IMC":
                    if sample["passage"] not in [None, ""]:
                        prompt += "文章: " + sample["passage"] + "\n"
                    prompt += "问题: " + sample["question"] + "\n"
                    for item in sample["options"]:
                        prompt += item + "\n"
                    if isinstance(sample["label"], str):
                        sample["label"] = [sample["label"]]

                    if self.CoT:
                        prompt += "回答: 让我们一步步思考。"
                    else:
                        prompt += "回答："

                elif self.type == "Cloze":
                    if sample["passage"] not in [None, ""]:
                        prompt += "文章: " + sample["passage"] + "\n"
                    prompt += "问题: " + sample["question"] + "\n"
                    if self.CoT:
                        prompt += "回答: 让我们一步步思考。"
                    else:
                        prompt += "回答："

            if num_tokens_from_messages(
                [prompt], self.completion_fn.model
            ) + 50 + self.max_tokens <= n_ctx_from_model_name(self.completion_fn.model):
                break
            self.OUT_OF_WINDOW = True
            if i == 0:
                prompt = "Once upon a time"
        return prompt, sample

    def post_process(self, sampled, sample):
        if self.CoT:
            if "So the answer is " in sampled:
                sampled = sampled.split("So the answer is ")[1]
            elif "所以答案是" in sampled:
                sampled = sampled.split("所以答案是")[1]
        if self.type == "MC":
            for i in range(26):
                if f"({chr(i+65)})" in sampled:
                    sampled = chr(i + 65)
                    break

            correct_answer = sample["label"]
        elif self.type == "Cloze":

            correct_answer = sample["answer"]
        else:
            if not isinstance(sample["label"], list):
                sample["label"] = [sample["label"]]
            correct_answer = ",".join(sample["label"])
            choice = []
            for i in range(26):
                if f"({chr(i+65)})" in sampled:
                    choice.append(chr(i + 65))
            sampled = ",".join(choice)

        return sampled, correct_answer

    def eval_sample(self, sample, rng):

        prompt, sample = self.pre_process(sample)

        result = self.completion_fn(
            prompt=prompt,
            temperature=self.temerpature,
            max_tokens=self.max_tokens,
        )

        sampled = result.get_completions()[0]
        sampled, correct_answer = self.post_process(sampled, sample)
        evals.record_and_check_match(
            prompt=prompt, sampled=sampled, expected=correct_answer
        )

    def eval_sample_batch(self, recorder, samples):
        data = [self.pre_process(item)[0] for item in samples]
        ideal = [self.pre_process(item)[1] for item in samples]
        results = self.completion_fn(
            inputs=data,
            temperature=self.temerpature,
            max_tokens=self.max_tokens,
        )
        processed_res, correct_answers = zip(
            *[self.post_process(results[i], ideal[i]) for i in range(len(results))]
        )

        for i in range(len(processed_res)):
            id = str(i)
            with recorder.as_default_recorder(id):
                record_sampling(prompt=data[i], sampled=results[i])
                evals.record_and_check_match(
                    prompt=data[i],
                    sampled=processed_res[i],
                    expected=correct_answers[i],
                )

    def run(self, recorder: RecorderBase):
        samples = evals.get_jsonl(self.sample_jsonl)
        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)
        return {
            "accuracy": evals.metrics.get_accuracy(recorder.get_events("match")),
            "OUT OF WINDOW": self.OUT_OF_WINDOW,
        }

evals/elsuite/dataset_specific/agieval.py
GPT-Fathom/GPT-Fathom
import json
import os
import random
from typing import Optional
from urllib.parse import parse_qs, urlparse

from datasets import load_dataset
from pydantic import BaseModel

import evals
import evals.metrics
from evals.api import CompletionFn
from evals.elsuite.utils import num_tokens_from_messages
from evals.formatting import make_abc
from evals.record import RecorderBase, record_sampling
from evals.registry import n_ctx_from_model_name



class BBH(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        task_name: str = None,
        samples_jsonl: str = False,
        CoT: bool = False,
        *args,
        instructions: Optional[str] = "",
        temerpature: float = 0.0,
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert (
            len(completion_fns) == 1
        ), "MultipleChoice only supports one completion fn"
        self.samples_jsonl = samples_jsonl
        path = os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            "../..",
            "registry",
            "prompt",
            "BBH",
            f"{task_name}.txt",
        )
        with open(path, "r") as file:
            if CoT:
                CoT_prompt = file.read().split("-----")[1]
            else:
                data = file.read().split("-----")[1].strip().split("\n\n")
                for i in range(len(data)):
                    if "So the answer is " in data[i]:
                        start = data[i].find("Let's think step by step.")
                        end = data[i].find("So the answer is ") + len(
                            ("So the answer is ")
                        )

                        data[i] = data[i][:start] + "The answer is " + data[i][end:]

                CoT_prompt = "\n\n".join(data)
        self.instructions = instructions
        self.few_shot_prompt = CoT_prompt
        self.temerpature = temerpature
        self.CoT = CoT
        self.OUT_OF_WINDOW = False

    def pre_process(self, sample):
        for i in range(3, -1, -1):
            few_shot_prompt = self.few_shot_prompt
            few_shot_prompt = few_shot_prompt.split("\n\nQ: ")
            few_shot_prompt = "\n\nQ: ".join(few_shot_prompt[: i + 1])

            prompt = few_shot_prompt + "\n\nQ: " + sample["input"] + "\nA: "
            if self.CoT:
                prompt += "Let's think step by step. "

            if num_tokens_from_messages(
                [prompt], self.completion_fn.model
            ) + 50 + 400 <= n_ctx_from_model_name(self.completion_fn.model):
                break
            self.OUT_OF_WINDOW = True

        return prompt, sample["ideal"]

    def post_process(self, sampled):
        if self.CoT:
            if "So the answer is " in sampled:
                sampled = sampled.split("So the answer is ")[1]
        else:
            if "The answer is " in sampled:
                sampled = sampled.split("The answer is ")[1]

        return sampled.strip()

    def eval_sample(self, sample, rng):
        prompt, correct_answer = self.pre_process(sample)

        result = self.completion_fn(
            prompt=prompt,
            temperature=self.temerpature,
            max_tokens=400,
        )

        sampled = result.get_completions()[0]
        sampled = self.post_process(sampled)

        evals.record_and_check_match(
            prompt=prompt,
            sampled=sampled,
            expected=correct_answer,
        )

    def eval_sample_batch(self, recorder, samples):
        data = [self.pre_process(item)[0] for item in samples]
        ideal = [self.pre_process(item)[1] for item in samples]
        results = self.completion_fn(
            inputs=data,
            temperature=self.temerpature,
            max_tokens=400,
        )
        processed_res = [self.post_process(item) for item in results]
        for i in range(len(processed_res)):
            id = str(i)
            with recorder.as_default_recorder(id):
                record_sampling(prompt=data[i], sampled=results[i])
                evals.record_and_check_match(
                    prompt=data[i],
                    sampled=processed_res[i],
                    expected=ideal[i],
                )

    def run(self, recorder: RecorderBase):
        samples = self.get_samples()

        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)
        return {
            "accuracy": evals.metrics.get_accuracy(recorder.get_events("match")),
            "OUT OF WINDOW": self.OUT_OF_WINDOW,
        }

evals/elsuite/dataset_specific/bbh.py
GPT-Fathom/GPT-Fathom
import os
import random
from typing import Optional
from urllib.parse import parse_qs, urlparse
from evals.record import record_sampling

from datasets import load_dataset, load_from_disk
from pydantic import BaseModel

import evals
import evals.metrics
from evals.api import CompletionFn
from evals.elsuite.utils import num_tokens_from_messages
from evals.formatting import make_abc
from evals.record import RecorderBase
from evals.registry import n_ctx_from_model_name


def parse_option(sampled):
    import re

    match = re.search(r"\(([A-Z])\)", sampled)
    if match:
        letter = match.group(1)
        return letter
    else:
        return sampled


class Sample(BaseModel):
    question: str
    answers: list[str]
    label: int
    CoT: str = None


def get_dataset(local, url: str) -> list[Sample]:
    parsed = urlparse(url)
    if parsed.scheme == "hf":
        query = parse_qs(parsed.query)
        query = {k: v[0] for k, v in query.items()}

        path = "ceval/ceval-exam"
        if not local:
            dataset = load_dataset(path, **query)
        else:
            path = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                "../..",
                "registry",
                "cached_hf_data",
                "ceval",
                query["name"],
                "test" if query["split"] == "val" else "fewshot",
            )
            dataset = load_from_disk(path)
        return [
            Sample(
                question=sample["question"],
                answers=[sample["A"], sample["B"], sample["C"], sample["D"]],
                label=ord(sample["answer"]) - ord("A"),
                CoT=sample["explanation"],
            )
            for sample in dataset
        ]


class CEVAL(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        dataset: str = None,
        num_few_shot: int = 0,
        few_shot: str = None,
        no_MC_prompt: bool = False,
        CoT: bool = False,
        Chinese: bool = True,
        max_tokens: int = 256,
        *args,
        temerpature: float = 0.0,
        instructions: Optional[str] = "",
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert (
            len(completion_fns) == 1
        ), "MultipleChoice only supports one completion fn"
        self.dataset = dataset
        self.instructions = instructions
        self.few_shot = num_few_shot
        self.few_shot_prompt = ""
        self.no_MC_prompt = no_MC_prompt
        self.temerpature = temerpature
        self.max_tokens = max_tokens
        self.Chinese = Chinese
        self.CoT = CoT
        if self.few_shot > 0:
            self.example = get_dataset(self.local_dataset, few_shot)
        self.OUT_OF_WINDOW = False

    def make_few_shot_prompt(self, num_shot):
        few_shot_prompt = ""
        rng = random.Random(47)
        for i in range((num_shot)):
            options, correct_answer, ans_ctx = make_abc(
                answers=self.example[i].answers,
                correct_idx=self.example[i].label,
                rng=rng,
            )
            few_shot_prompt += "Q: " + self.example[i].question + "\n\n" + options  #
            if not self.no_MC_prompt:
                few_shot_prompt += "\n请只输出正确答案对应的字母编号\n"
            else:
                few_shot_prompt += "\n"
            if self.CoT:
                few_shot_prompt += (
                    "A: "
                    + "让我们一步一步思考。"
                    + self.example[i].CoT
                    + "所以答案是 "
                    + ans_ctx
                    + "\n\n"
                )
            else:
                few_shot_prompt += "A: " + ans_ctx + "\n\n"

        return few_shot_prompt
    def pre_process(self, sample): 
        options, correct_answer, _ = make_abc(
            answers=sample.answers,
            correct_idx=sample.label,
            rng=random.Random(47),
        )
        for i in range(self.few_shot, -1, -1):
            few_shot_prompt = self.make_few_shot_prompt(i)
            prompt = (
                self.instructions
                + "\n\n"
                + few_shot_prompt
                + "Q: "
                + sample.question
                + "\n\n"
                + options
            )
            if not self.no_MC_prompt:
                prompt += "\n请只输出正确答案对应的字母编号\nA: "
            else:
                prompt += "\nA: "
            if self.CoT:
                prompt += "让我们一步一步思考。"

            if num_tokens_from_messages(
                [prompt], self.completion_fn.model
            ) + 50 + self.max_tokens <= n_ctx_from_model_name(self.completion_fn.model):
                break
            self.OUT_OF_WINDOW = True
        return prompt, correct_answer


    def eval_sample(self, sample, rng):
        assert isinstance(sample, Sample)

        prompt, correct_answer = self.pre_process(sample)

        result = self.completion_fn(
            prompt=prompt,
            temperature=self.temerpature,
            max_tokens=self.max_tokens if self.CoT else 10,
        )
        sampled = result.get_completions()[0]
        if "所以答案是 " in sampled:
            sampled = sampled.split("所以答案是 ")[1]
        sampled = parse_option(sampled)

        evals.record_and_check_match(
            prompt=prompt,
            sampled=sampled.lstrip(),
            expected=correct_answer,
        )

    def eval_sample_batch(self, recorder, samples):
        data = [self.pre_process(item)[0] for item in samples]
        ideal = [self.pre_process(item)[1] for item in samples]
        results = self.completion_fn(inputs=data, temperature=self.temerpature, max_tokens=50)
        processed_res = [(item) for item in results]
        for i in range(len(processed_res)):
            correct_answer = ideal[i]
            sampled = processed_res[i]
            if "所以答案是 " in sampled:
                sampled = sampled.split("所以答案是 ")[1]
            sampled = parse_option(sampled)
            id = str(i)
            with recorder.as_default_recorder(id):
                record_sampling(prompt=data[i], sampled=results[i])
                evals.record_and_check_match(
                    prompt=data[i],
                    sampled=sampled.lstrip(),
                    expected=correct_answer,
                )

    def run(self, recorder: RecorderBase):
        samples = get_dataset(self.local_dataset, self.dataset)

        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)

        print("OUT OF WINDOW: ", self.OUT_OF_WINDOW)
        return {
            "accuracy": evals.metrics.get_accuracy(recorder.get_events("match")),
            "OUT OF WINDOW": self.OUT_OF_WINDOW,
        }

evals/elsuite/dataset_specific/ceval.py
GPT-Fathom/GPT-Fathom
"""The eval script for lambada dataset"""
import os
from typing import Any

from datasets import load_dataset

import evals
import evals.metrics
from evals.api import CompletionFn
from evals.prompt.base import chat_prompt_to_text_prompt, is_chat_prompt
from evals.record import RecorderBase, record_sampling



CHAT_MODELS = {
    "gpt-3.5-turbo",
    "gpt-3.5-turbo-0301",
    "gpt-3.5",
    "gpt-4",
    "gpt-4-0314",
    "gpt-4-32k",
    "gpt-4-32k-0314",
}


def extract_first_word(text):
    import re

    text = text.strip()
    if text.startswith("A:"):
        text = text[2:]
    pattern = r"\b[a-zA-Z]+\b"  # 匹配一个由字母和数字组成的单词
    match = re.search(pattern, text)
    if match:
        return match.group()
    else:
        return text


class Lambada(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        samples_jsonl: str,
        num_few_shot: int = 0,
        few_shot_jsonl: str = None,
        temerpature: float = 0.0,
        *args,
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert (
            len(completion_fns) == 1
        ), "MultipleChoice only supports one completion fn"
        self.samples_jsonl = samples_jsonl
        self.num_few_shot = num_few_shot
        self.temerpature = temerpature
        if self.num_few_shot > 0:
            assert (
                few_shot_jsonl is not None
            ), "few shot requires few shot sample dataset"
            self.few_shot_jsonl = few_shot_jsonl
            self.few_shot = evals.get_jsonl(self.few_shot_jsonl)

    def prompt_to_fewshot(self, sample):
        prompt = sample["input"]
        if self.num_few_shot > 0:
            assert is_chat_prompt(sample["input"]), "few shot requires chat prompt"
            prompt = sample["input"][:-1]
            for s in self.few_shot[: self.num_few_shot]:
                prompt += s["sample"]
            prompt += sample["input"][-1:]
        return prompt

    """
        Turn the sample into a prompt
    """

    def pre_process(self, sample):
        prompt = self.prompt_to_fewshot(sample)
        # Handle the '\n' relevant problems
        if self.completion_fns[0].model not in CHAT_MODELS:
            if self.num_few_shot == 0:
                prompt = chat_prompt_to_text_prompt(
                    prompt, Q_A_ENDING=False, new_line_end=False
                )
            else:
                prompt = chat_prompt_to_text_prompt(
                    prompt, Q_A_ENDING=True, new_line_end=False
                )
        return prompt, sample["ideal"]

    """
        Output post processing
    """

    def post_process(self, res):
        res = res.lstrip()
        res = extract_first_word(res)
        return res

    def eval_sample(self, sample: Any, *_):
        prompt, ideal = self.pre_process(sample)
        result = self.completion_fn(
            prompt=prompt,
            temperature=self.temerpature,
            max_tokens=15,
        )
        sampled = self.post_process(result.get_completions()[0])
        evals.record_and_check_match(
            prompt=prompt,
            sampled=sampled,
            expected=ideal,
        )

    def eval_sample_batch(self, recorder, samples):
        data = [self.pre_process(item)[0] for item in samples]
        ideal = [self.pre_process(item)[1] for item in samples]
        results = self.completion_fn(
            inputs=data,
            temperature=self.temerpature,
            max_tokens=8,
        )
        processed_res = [self.post_process(item) for item in results]
        for i in range(len(processed_res)):
            id = str(i)
            with recorder.as_default_recorder(id):
                record_sampling(prompt=data[i], sampled=results[i])
                evals.record_and_check_match(
                    prompt=data[i],
                    sampled=processed_res[i],
                    expected=ideal[i],
                )

    def run(self, recorder: RecorderBase):
        samples = self.get_samples()
        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)
        return {
            "accuracy": evals.metrics.get_accuracy(recorder.get_events("match")),
        }

evals/elsuite/dataset_specific/lambada.py
GPT-Fathom/GPT-Fathom
import json
import os
import random
from typing import Optional
from urllib.parse import parse_qs, urlparse

from datasets import load_dataset, load_from_disk
from pydantic import BaseModel

import evals
import evals.metrics
from evals.api import CompletionFn
from evals.elsuite.utils import num_tokens_from_messages
from evals.formatting import make_abc
from evals.record import RecorderBase, record_sampling
from evals.registry import n_ctx_from_model_name


class Sample(BaseModel):
    question: str
    answers: list[str]
    label: int


def parse_option(sampled):
    import re

    match = re.search(r"\(([A-Z])\)", sampled)
    if match:
        letter = match.group(1)
        return letter
    else:
        return sampled


def get_dataset(local, url: str) -> list[Sample]:
    parsed = urlparse(url)
    if parsed.scheme == "hf":
        query = parse_qs(parsed.query)
        query = {k: v[0] for k, v in query.items()}

        path = parsed.netloc
        if not local:
            dataset = load_dataset(path, **query)
        if path == "hendrycks_test":
            if local:
                path = os.path.join(
                    os.path.dirname(os.path.abspath(__file__)),
                    "../..",
                    "registry",
                    "cached_hf_data",
                    "mmlu",
                    query["name"],
                    "test" if query["split"] == "test" else "fewshot",
                )
                dataset = load_from_disk(path)
            return [
                Sample(
                    question=sample["question"],
                    answers=sample["choices"],
                    label=sample["answer"],
                )
                for sample in dataset
            ]
    raise ValueError(f"Unknown question dataset {url}")


class MC_CoT(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        dataset: str = None,
        no_MC_prompt: bool = False,
        CoT: bool = False,
        task_name: str = None,
        temerpature: float = 0.0,
        *args,
        instructions: Optional[str] = "",
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert (
            len(completion_fns) == 1
        ), "MultipleChoice only supports one completion fn"
        self.dataset = dataset
        self.temerpature = temerpature
        path = os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            "../..",
            "registry",
            "prompt",
            "mmlu",
            "mmlu-cot.json",
        )
        with open(path, "r") as file:
            CoT_prompt = json.load(file)
        self.instructions = CoT_prompt[task_name]
        self.few_shot_prompt = CoT_prompt[task_name]
        self.no_MC_prompt = no_MC_prompt
        self.CoT = CoT
        self.max_tokens = 200
        self.OUT_OF_WINDOW = False

    """
        Turn the sample into a prompt
    """

    def pre_process(self, sample):
        options, correct_answer, _ = make_abc(
            answers=sample.answers,
            correct_idx=sample.label,
            rng=random.Random(47),
        )
        for i in range(5, -1, -1):
            few_shot_prompt = self.few_shot_prompt
            few_shot_prompt = few_shot_prompt.split("\n\nQ: ")
            few_shot_prompt = "\n\nQ: ".join(few_shot_prompt[: i + 1])
            prompt = few_shot_prompt + "Q: " + sample.question + "\n\n" + options
            if not self.no_MC_prompt:
                prompt += "\nPlease answer with the letter of the correct answer.\nA: "
            else:
                prompt += "\nA: "
            if self.CoT:
                prompt += "Let's think step by step. "
            if num_tokens_from_messages(
                [prompt], self.completion_fn.model
            ) + 50 + self.max_tokens <= n_ctx_from_model_name(self.completion_fn.model):
                break
            self.OUT_OF_WINDOW = True

        return prompt, correct_answer

    def eval_sample(self, sample, rng):
        assert isinstance(sample, Sample)

        # print(prompt)
        prompt, correct_answer = self.pre_process(sample)
        result = self.completion_fn(
            prompt=prompt,
            temperature=self.temerpature,
            max_tokens=self.max_tokens,
        )
        sampled = result.get_completions()[0]
        if "The answer is " in sampled:
            sampled = sampled.split("The answer is ")[1]
        sampled = parse_option(sampled)

        evals.record_and_check_match(
            prompt=prompt,
            sampled=sampled,
            expected=correct_answer,
        )

    def eval_sample_batch(self, recorder, samples):
        data = [self.pre_process(item)[0] for item in samples]
        ideal = [self.pre_process(item)[1] for item in samples]
        results = self.completion_fn(
            inputs=data,
            temperature=self.temerpature,
            max_tokens=self.max_tokens,
        )
        processed_res = [
            sampled.split("The answer is ")[1]
            if "The answer is " in sampled
            else sampled
            for sampled in results
        ]
        processed_res = [parse_option(sampled) for sampled in processed_res]
        for i in range(len(processed_res)):
            id = str(i)
            with recorder.as_default_recorder(id):
                record_sampling(prompt=data[i], sampled=results[i])
                evals.record_and_check_match(
                    prompt=data[i],
                    sampled=processed_res[i],
                    expected=ideal[i],
                )

    def run(self, recorder: RecorderBase):
        samples = get_dataset(self.local_dataset, self.dataset)

        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)
        return {
            "accuracy": evals.metrics.get_accuracy(recorder.get_events("match")),
            "OUT OF WINDOW": self.OUT_OF_WINDOW,
        }

evals/elsuite/dataset_specific/mmlu_CoT.py
GPT-Fathom/GPT-Fathom
import backoff
import googleapiclient
import numpy as np
from googleapiclient import discovery

import evals
from evals.api import CompletionFn
from evals.elsuite import utils
from evals.prompt.base import chat_prompt_to_text_prompt
from evals.record import RecorderBase, record_sampling


@backoff.on_exception(
    wait_gen=backoff.expo,
    exception=(googleapiclient.errors.HttpError,),
    max_value=60,
    factor=1.5,
)
def perspective(input):

    API_KEY = None
    assert API_KEY != None, "Replace it with your perspective API key!"

    client = discovery.build(
        "commentanalyzer",
        "v1alpha1",
        developerKey=API_KEY,
        discoveryServiceUrl="https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1",
        static_discovery=False,
    )
    if input == "":
        input = "a"

    analyze_request = {
        "comment": {"text": input},
        "languages": ["en"],
        "requestedAttributes": {"TOXICITY": {}},
    }

    response = client.comments().analyze(body=analyze_request).execute()
    return response["attributeScores"]["TOXICITY"]["summaryScore"]["value"]


class RealToxicity(evals.Eval):
    def __init__(
        self,
        completion_fns: list[CompletionFn],
        samples_jsonl: str,
        *args,
        max_tokens: int = 100,
        temerpature: float = 0.0,
        **kwargs,
    ):
        super().__init__(completion_fns, *args, **kwargs)
        assert len(completion_fns) == 1, "FuzzyMatch only supports one completion fn"
        self.max_tokens = max_tokens
        self.temerpature =temerpature
        self.samples_jsonl = samples_jsonl

    def eval_sample(self, test_sample, rng):
        del rng

        assert isinstance(test_sample, dict), "sample must be a dict"

        prompt = test_sample["input"]

        result = self.completion_fn(
            prompt=prompt,
            temperature=self.temerpature,  # Q: why are these hardcoded?
            max_tokens=self.max_tokens,
        )
        sampled = result.get_completions()[0]

        evals.record.record_metrics(
            summaryscore=perspective(sampled),
        )

    def eval_sample_batch(self, recorder, samples):
        data = [chat_prompt_to_text_prompt(item["input"]) for item in samples]
        # data = data[:10]
        results = self.completion_fn(
            inputs=data,
            temperature=self.temerpature,
            max_tokens=self.max_tokens,
        )
        processed_res = [(item) for item in results]
        for i in range(len(processed_res)):
            id = str(i)
            with recorder.as_default_recorder(id):
                record_sampling(prompt=data[i], sampled=results[i])
                evals.record.record_metrics(
                    summaryscore=perspective(results[i]),
                )

    def run(self, recorder: RecorderBase):
        samples = self.get_samples()
        if self.eval_in_batch == False:
            self.eval_all_samples(recorder, samples)
        else:
            self.eval_sample_batch(recorder, samples)

        return {
            "summaryscore": np.mean(recorder.get_scores("summaryscore")),
        }

evals/elsuite/dataset_specific/realtoxic.py
GPT-Fathom/GPT-Fathom
from typing import TYPE_CHECKING, Optional, Union

from evals.prompt.base import OpenAICreateChatPrompt

if TYPE_CHECKING:
    from dataclasses import dataclass
else:
    from pydantic.dataclasses import dataclass


@dataclass
class ModelGradedSpec:
    # must have
    prompt: Union[str, OpenAICreateChatPrompt]
    choice_strings: Union[list[str], str]
    input_outputs: dict[str, str]

    # optional
    eval_type: Optional[str] = None
    choice_scores: Optional[Union[dict[str, float], str]] = None
    output_template: Optional[str] = None

    # unused
    key: Optional[str] = None
    group: Optional[str] = None

evals/elsuite/modelgraded/base.py
GPT-Fathom/GPT-Fathom
"""
Generic eval that uses a prompt + classification.
"""
from collections import Counter
from random import Random
from typing import Any, Optional, Union

import evals
import evals.record
from evals.elsuite.modelgraded.classify_utils import (
    classify,
    sample_and_concat_n_completions,
)
from evals.elsuite.utils import PromptFn, scrub_formatting_from_prompt


class ModelBasedClassify(evals.Eval):
    def __init__(
        self,
        modelgraded_spec: str,
        *args,
        modelgraded_spec_args: Optional[dict[str, dict[str, str]]] = None,
        sample_kwargs: Optional[dict[str, Any]] = None,
        eval_kwargs: Optional[dict[str, Any]] = None,
        multicomp_n: Union[int, str] = 1,
        eval_type: Optional[str] = None,
        metaeval: bool = False,
        **kwargs,
    ):
        super().__init__(*args, **kwargs)
        # treat last completion_fn as eval_completion_fn
        self.eval_completion_fn = self.completion_fns[-1]
        if len(self.completion_fns) > 1:
            self.completion_fns = self.completion_fns[:-1]
        n_models = len(self.completion_fns)
        self.sample_kwargs = {"max_tokens": 1024}
        self.sample_kwargs.update(sample_kwargs or {})
        self.eval_kwargs = {"max_tokens": 1024}
        self.eval_kwargs.update(eval_kwargs or {})
        self.metaeval = metaeval
        self.modelgraded_spec_args = modelgraded_spec_args or {}
        self.eval_type = eval_type
        if multicomp_n == "from_models":
            assert n_models > 1
            self.multicomp_n = n_models
        else:
            assert isinstance(multicomp_n, int)
            self.multicomp_n = multicomp_n
        if len(self.completion_fns) > 1:
            assert self.multicomp_n == n_models

        self.mg = self.registry.get_modelgraded_spec(modelgraded_spec)

    def eval_sample(self, test_sample: dict, rng: Random) -> None:
        """Evaluate a single sample.

        Recorded metrics are always: one of the self.choice_strings, or "__invalid__".
        """
        # process test_sample
        for k in self.mg.input_outputs:
            test_sample[k] = scrub_formatting_from_prompt(test_sample[k])

        # run policy completions
        completions = {}
        for k, v in self.mg.input_outputs.items():
            if v in test_sample:  # test_sample already has completion, skip.
                continue
            if self.multicomp_n > 1:
                completion = sample_and_concat_n_completions(
                    self.completion_fns,
                    prompt=test_sample[k],
                    template_i=self.mg.output_template,
                    sample_kwargs=self.sample_kwargs,
                    n=self.multicomp_n,
                )
            else:
                get_input_completion = PromptFn(
                    test_sample[k],
                    completion_fn=self.completion_fn,
                    **self.sample_kwargs,
                )
                completion, _ = get_input_completion()
            completions[v] = completion

        # run modelgraded eval
        metrics = {}
        choice, info = classify(
            mg=self.mg,
            completion_fn=self.eval_completion_fn,
            completion_kwargs=self.eval_kwargs,
            eval_type=self.eval_type,
            n=self.multicomp_n,
            format_kwargs={**completions, **test_sample, **self.modelgraded_spec_args},
        )
        metrics.update(dict(choice=choice, score=info["score"]))

        # run metaeval if requested
        if self.metaeval:
            assert "choice" in test_sample
            metrics["metascore"] = choice == test_sample["choice"]

        evals.record.record_metrics(**metrics)

        return choice

    def run(self, recorder):
        samples = self.get_samples()

        self.eval_all_samples(recorder, samples)
        record_metrics = {}

        all_sample_metrics = recorder.get_metrics()
        if not all_sample_metrics:
            return record_metrics

        # record the counts
        choices = [m["choice"] for m in all_sample_metrics]
        counts = dict(Counter(choices))
        record_metrics.update({f"counts/{k}": v for k, v in counts.items()})

        # record the scores
        scores = [m["score"] for m in all_sample_metrics if m["score"] is not None]
        if scores:
            record_metrics[f"score"] = sum(scores) / len(scores)
        metascores = [m["metascore"] for m in all_sample_metrics if "metascore" in m]
        if metascores:
            record_metrics[f"metascore"] = sum(metascores) / len(metascores)

        return record_metrics

evals/elsuite/modelgraded/classify.py
GPT-Fathom/GPT-Fathom
import logging
import string
from typing import Any, Callable, Iterable, Optional, Union

from evals import CompletionFn
from evals.elsuite.modelgraded.base import ModelGradedSpec
from evals.elsuite.utils import PromptFn, format_necessary, format_prompt
from evals.prompt.base import OpenAICreateChatPrompt, is_chat_prompt

INVALID_STR = "__invalid__"


ANSWER_PROMPTS = {
    # e.g. "Yes"
    "classify": "Answer the question by printing only a single choice from {choices} (without quotes or punctuation) corresponding to the correct answer with no other text.".strip(),
    # e.g. "Yes\n The reasons are: ..."
    "classify_cot": "First, answer by printing a single choice from {choices} (without quotes or punctuation) corresponding to the correct answer. Then, from the next line, explain your reasonings step by step.".strip(),
    # e.g. "Let's think step by step. ...\nYes"
    "cot_classify": """
First, write out in a step by step manner your reasoning to be sure that your conclusion is correct. Avoid simply stating the correct answer at the outset. Then print only a single choice from {choices} (without quotes or punctuation) on its own line corresponding to the correct answer. At the end, repeat just the answer by itself on a new line.

Reasoning:""".strip(),
    "cot_classify_jp": """
まず、一歩一歩あなたの推論を書き出してください。単に正しい答えを最初に述べることを避けてください。次に、{choices}（引用符や句読点なし）から正しい答えに対応する1つの選択肢を単独の行に書きだしてください。最後に、答えだけを新しい行に繰り返してください。

推論：
    """.strip(),
}
MATCH_FNS = {
    "include": lambda x, y: float(x in y),
    "exact": lambda x, y: float(x == y),
    "endswith": lambda x, y: x.endswith(y),
    "starts_or_endswith": lambda x, y: x.startswith(y) or x.endswith(y),
}


def get_choice_strings(choice_strings: Union[list[str], str], n: Optional[int] = None):
    # 'choice_strings' is a list of strings that specifies the possible choices
    if choice_strings == "from_n":
        choice_strings = [str(i + 1) for i in range(n)]
    elif choice_strings == "from_n_abc":
        choice_strings = [string.ascii_lowercase[i % 26] for i in range(n)]
    elif choice_strings == "from_n_ABC":
        choice_strings = [string.ascii_uppercase[i % 26] for i in range(n)]
    # make sure each choice doesn't contain any punctuation
    for s in choice_strings:
        assert not any(c in s for c in string.punctuation), f"{s} contains punctuation"
    return choice_strings


def classify(
    mg: ModelGradedSpec,
    completion_fn: CompletionFn,
    completion_kwargs: Optional[dict[str, Any]] = None,
    format_kwargs: Optional[dict[str, Any]] = None,
    eval_type: Optional[str] = None,
    n: Optional[int] = None,
    match_fn: str = "starts_or_endswith",
) -> str:
    completion_kwargs = completion_kwargs or {}
    format_kwargs = format_kwargs or {}

    # get choice strings
    choice_strings = get_choice_strings(mg.choice_strings, n=n)

    # append answer prompt
    prompt = mg.prompt
    if isinstance(prompt, str):
        prompt = [{"role": "user", "content": prompt}]
    if eval_type:
        prompt = append_answer_prompt(
            prompt=prompt,
            eval_type=eval_type,
            choice_strings=choice_strings,
        )

    evaluate = PromptFn(prompt, completion_fn=completion_fn, **completion_kwargs)
    evaluation, prompt = evaluate(n=n, **format_kwargs)
    choice = get_choice(evaluation, mg.eval_type or eval_type, match_fn, choice_strings)
    score = get_choice_score(choice, choice_strings, mg.choice_scores)
    return choice, dict(
        score=score,
        sampled=[evaluation],
        prompt=prompt,
        invalid_choice=choice == INVALID_STR,
    )


def get_choice_score(
    choice: str,
    choice_strings: Iterable[str],
    choice_scores: Optional[Union[dict[str, float], str]] = None,
) -> Optional[float]:
    if choice_scores is None:
        return None
    if choice_scores == "from_strings":
        choice_scores = {c: float(c) for c in choice_strings}
    # assumption: each INVALID_STR contributes the lowest score
    if choice == INVALID_STR:
        return min(choice_scores.values())
    return choice_scores[choice]


def choice_to_str(choice_strings: Iterable[str]) -> str:
    """Return a string of choices, e.g. '"Yes" or "No" or "Maybe"'."""
    return " or ".join(f'"{choice}"' for choice in choice_strings)


def get_choice(
    text: str,
    eval_type: str,
    match_fn: Union[str, Callable],
    choice_strings: Iterable[str],
) -> str:
    """Clean the answer string to a choice string to one of choice_strings. Return '__invalid__.' if no match."""
    if isinstance(match_fn, str):
        match_fn = MATCH_FNS[match_fn]
    lines = text.strip().split("\n")
    if eval_type.startswith("cot_classify"):
        lines = lines[::-1]  # reverse lines
    for line in lines:
        line = line.strip()
        line = "".join(c for c in line if c not in string.punctuation)
        if not line:
            continue
        for choice in choice_strings:
            if match_fn(line, choice):
                return choice
    logging.warn(f"Choices {choice_strings} not parsable for {eval_type}: {text}")
    return INVALID_STR


def append_answer_prompt(
    prompt: OpenAICreateChatPrompt,
    eval_type: str,
    append_type: str = "as_content",
    answer_prompt: Optional[OpenAICreateChatPrompt] = None,
    choice_strings: Optional[Iterable[str]] = None,
) -> OpenAICreateChatPrompt:
    """Append answer prompt to prompt."""
    answer_prompt = answer_prompt or ANSWER_PROMPTS[eval_type]
    answer_prompt = format_prompt(answer_prompt, choices=choice_to_str(choice_strings))
    if append_type == "as_content":
        assert isinstance(
            answer_prompt, str
        ), f"prompt must be str, not {type(answer_prompt)}"
        prompt[-1]["content"] += "\n\n" + answer_prompt
    elif append_type == "as_message":
        assert is_chat_prompt(
            answer_prompt
        ), f"prompt must be chat prompt, not {answer_prompt}"
        prompt += answer_prompt
    else:
        raise ValueError(
            f"append_type must be 'as_content' or 'as_message', not {append_type}"
        )
    return prompt


def sample_and_concat_n_completions(
    completion_fns: list[CompletionFn],
    prompt: OpenAICreateChatPrompt,
    n: int,
    template_i: str,
    sample_kwargs: dict,
):
    assert template_i
    completion_i_s = []
    for i in range(n):
        if len(completion_fns) > 1:
            # use a separate model for each completion
            assert len(completion_fns) == n
            completion_fn = completion_fns[i]
        else:
            # use the single model for all completions
            completion_fn = completion_fns[0]
        get_input_completion = PromptFn(
            prompt, completion_fn=completion_fn, **sample_kwargs
        )
        completion_i, _ = get_input_completion()
        completion_i_s.append(completion_i)
    return concat_n_completions(completion_i_s, template_i=template_i)


def concat_n_completions(completions: Iterable[str], template_i: str) -> str:
    """Concatenate n completions into a single text string."""
    completion = ""
    for i, completion_i in enumerate(completions):
        completion += format_necessary(
            template_i,
            i=i + 1,
            i_abc=string.ascii_lowercase[i % 26],
            i_ABC=string.ascii_uppercase[i % 26],
            output=completion_i,
            n=len(completions),
        )
    return completion.strip()

evals/elsuite/modelgraded/classify_utils.py
GPT-Fathom/GPT-Fathom
from evals.elsuite.basic.match import Match


class TestMatch(Match):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs, samples_jsonl="")

    def get_samples(self):
        return [
            {
                "input": [
                    {
                        "role": "system",
                        "content": "Complete the phrase as concisely as possible.",
                    },
                    {"role": "user", "content": "Once upon a "},
                ],
                "ideal": "time",
            },
            {
                "input": [
                    {
                        "role": "system",
                        "content": "Complete the phrase as concisely as possible.",
                    },
                    {"role": "user", "content": "The first US president was "},
                ],
                "ideal": "George Washington",
            },
            {
                "input": [
                    {
                        "role": "system",
                        "content": "Complete the phrase as concisely as possible.",
                    },
                    {"role": "user", "content": "OpenAI was founded in 20"},
                ],
                "ideal": "15",
            },
        ]

evals/elsuite/test/match.py
GPT-Fathom/GPT-Fathom
import copy
import re
import string
from collections import Counter, defaultdict
from typing import Optional, Union

from transformers import LlamaTokenizerFast

from evals import CompletionFn
from evals.prompt.base import (
    OpenAICreateChatPrompt,
    OpenAICreatePrompt,
    Prompt,
    chat_prompt_to_text_prompt,
    is_chat_prompt,
)


def get_answer(text, answer_prompt, ignore_case=False):
    if ignore_case:
        idx = text.lower().rfind(answer_prompt.lower())
    else:
        idx = text.rfind(answer_prompt)

    if idx == -1:
        return None
    return text[idx : idx + len(answer_prompt)]


def get_consensus(answers):
    counts = defaultdict(int)
    for answer in answers:
        counts[answer] += 1
    counts[None] = 0
    return max(counts, key=counts.get)


def normalize(s: str) -> str:
    """Lower text and remove punctuation, articles and extra whitespace."""
    s = s.lower()
    exclude = set(string.punctuation)
    s = "".join(char for char in s if char not in exclude)
    s = re.sub(r"\b(a|an|the)\b", " ", s)
    s = " ".join(s.split())
    return s


def fuzzy_match(s1: str, s2: str) -> bool:
    s1 = normalize(s1)
    s2 = normalize(s2)

    if s1 == "" or s2 == "":
        return s1 == s2

    return s1 in s2 or s2 in s1


def get_scores_from_text(text: str) -> dict:
    pattern = r"## (.+?)\n.+?(\d)/5"
    matches = re.findall(pattern, text, re.DOTALL)
    return {k: int(v) for k, v in dict(matches).items()}


def get_yesno_from_text(text: str) -> dict:
    pattern = r"## (.+?)\n.+?([yn])"
    matches = re.findall(pattern, text, re.DOTALL)
    return {k: v for k, v in dict(matches).items()}


def get_letter_from_data(data: str) -> str:
    last_y = (data.rfind("y"), "y")
    last_n = (data.rfind("n"), "n")
    char = max(last_y, last_n)[1]
    return char


def f1_score(prediction: str, answers: list[str]) -> float:
    def _f1_score(prediction: str, ground_truth: str):
        prediction_tokens = normalize(prediction).split()
        ground_truth_tokens = normalize(ground_truth).split()
        common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
        num_same = sum(common.values())
        if num_same == 0:
            return 0
        precision = 1.0 * num_same / len(prediction_tokens)
        recall = 1.0 * num_same / len(ground_truth_tokens)
        f1 = (2 * precision * recall) / (precision + recall)
        return f1

    return max([_f1_score(prediction, answer) for answer in answers])


def scrub_formatting_from_prompt(prompt):
    scrubbed_prompt = copy.copy(prompt)

    if is_chat_prompt(prompt):
        for i, msg in enumerate(scrubbed_prompt):
            if "content" in msg:
                scrubbed_prompt[i]["content"] = (
                    msg["content"].replace("{", "{{").replace("}", "}}")
                )
    else:
        scrubbed_prompt = scrubbed_prompt.replace("{", "{{").replace("}", "}}")
    return scrubbed_prompt


def format_necessary(
    template: str, allow_missing: bool = False, **kwargs: dict[str, str]
) -> str:
    """Format a template string with only necessary kwargs."""
    keys = [k[1] for k in string.Formatter().parse(template) if k[1]]
    if allow_missing:
        assert (
            len([k for k in keys if k in kwargs]) > 0
        ), f"Required: {keys}, got: {sorted(kwargs)}, no inputs are used.\nTemplate:\n{template}"
        cur_keys = {k: kwargs.get(k, "{" + k + "}") for k in keys}
    else:
        assert all(
            k in kwargs for k in keys
        ), f"Required: {keys}, got: {sorted(kwargs)}.\nTemplate:\n{template}"
        cur_keys = {k: kwargs[k] for k in keys}
    return template.format(**cur_keys)


def format_prompt(
    prompt: OpenAICreatePrompt, allow_missing: bool = False, **kwargs: dict[str, str]
) -> OpenAICreatePrompt:
    """Format a prompt with only necessary kwargs."""
    # if any input kwargs is chat prompt, convert to text prompt
    kwargs = {
        k: chat_prompt_to_text_prompt(v, for_completion=False)
        if is_chat_prompt(v)
        else v
        for k, v in kwargs.items()
    }
    if is_chat_prompt(prompt):
        new_prompt = []
        for msg in prompt:
            formatted_msg = copy.copy(msg)
            if "content" in formatted_msg:
                formatted_msg["content"] = format_necessary(
                    formatted_msg["content"], allow_missing=allow_missing, **kwargs
                )
            new_prompt.append(formatted_msg)
        prompt = new_prompt
    else:
        # Prompt is a string
        prompt = format_necessary(prompt, allow_missing=allow_missing, **kwargs)
    return prompt


class PromptFn:
    """
    Wrap calls to a completion_fn with a prompt template with applicable keyword args.
    This will pass many args relevant to OpenAI Completion API, may be ignored by other completion_fn.
    """

    def __init__(
        self,
        prompt: Union[OpenAICreatePrompt, OpenAICreateChatPrompt, Prompt],
        completion_fn: CompletionFn,
        max_tokens: int,
        temperature: int = 0,
        n_samples: Optional[int] = None,
        completion_kwargs: Optional[dict] = {},
    ):
        self.prompt = prompt
        self.max_tokens = max_tokens
        self.completion_fn = completion_fn
        self.temperature = temperature
        self.completion_kwargs = completion_kwargs
        self.n_samples = n_samples

    def __call__(self, **kwargs):
        # if any input kwargs is chat prompt, convert to text prompt
        kwargs = {
            k: chat_prompt_to_text_prompt(v, for_completion=False)
            if is_chat_prompt(v)
            else v
            for k, v in kwargs.items()
        }
        if is_chat_prompt(self.prompt):
            prompt = []
            for msg in self.prompt:
                formatted_msg = copy.copy(msg)
                if "content" in formatted_msg:
                    formatted_msg["content"] = format_necessary(
                        formatted_msg["content"], **kwargs
                    )
                prompt.append(formatted_msg)
        else:
            # Prompt is a string
            prompt = format_necessary(self.prompt, **kwargs)

        result = self.completion_fn(
            prompt=prompt,
            max_tokens=self.max_tokens,
            temperature=self.temperature,
            top_p=1,
            frequency_penalty=0,
            presence_penalty=0,
            n=(1 if self.n_samples is None else self.n_samples),
            **self.completion_kwargs,
        )
        sampled = result.get_completions()[0]
        return sampled, prompt


import tiktoken


def num_tokens_from_messages(messages, model="gpt-3.5-turbo-0301"):
    """Returns the number of tokens used by a list of messages."""
    try:
        encoding = tiktoken.encoding_for_model(model)
    except KeyError:
        if model in ["davinci", "davinci-instruct-beta", "text-davinci-001"]:
            encoding = tiktoken.get_encoding("r50k_base")
        elif model in [
            "text-davinci-002",
            "text-davinci-003",
            "code-cusman-001",
            "code-davinci-edit-001",
        ]:
            encoding = tiktoken.get_encoding("p50k_base")
        elif model.startswith("llama"):
            encoding = LlamaTokenizerFast.from_pretrained(
                "meta-llama/Llama-2-70b-hf", use_fast=False, use_auth_token=True
            )
        else:
            encoding = tiktoken.get_encoding("cl100k_base")
    if model == "gpt-3.5-turbo" or model == "gpt-3.5":
        # print("Warning: gpt-3.5-turbo may change over time. Returning num tokens assuming gpt-3.5-turbo-0301.")
        return num_tokens_from_messages(messages, model="gpt-3.5-turbo-0301")
    elif model == "gpt-4":
        # print("Warning: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.")
        return num_tokens_from_messages(messages, model="gpt-4-0314")
    elif model == "gpt-3.5-turbo-0301":
        tokens_per_message = (
            4  # every message follows <|start|>{role/name}\n{content}<|end|>\n
        )
        tokens_per_name = -1  # if there's a name, the role is omitted
    elif model == "gpt-4-0314":
        tokens_per_message = 3
        tokens_per_name = 1
    else:
        tokens_per_message = 0
        tokens_per_name = 0
    num_tokens = 0

    for message in messages:
        delta = 0
        num_tokens += tokens_per_message

        num_tokens += len(encoding.encode(message))

    return num_tokens

evals/elsuite/utils.py
GPT-Fathom/GPT-Fathom
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This code is adapted from OpenAI's release
# https://github.com/openai/human-eval/blob/master/human_eval/execution.py

import contextlib
import faulthandler
import io
import itertools
import multiprocessing
import os
import platform
import signal
import tempfile
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed

import numpy as np


def check_correctness(check_program, timeout, task_id, completion_id):
    """
    Evaluates the functional correctness of a completion by running the test
    suite provided in the problem.

    :param completion_id: an optional completion ID so we can match
        the results later even if execution finishes asynchronously.
    """
    manager = multiprocessing.Manager()
    result = manager.list()

    p = multiprocessing.Process(
        target=unsafe_execute, args=(check_program, result, timeout)
    )
    p.start()
    p.join(timeout=timeout + 1)
    if p.is_alive():
        p.kill()

    if not result:
        result.append("timed out")

    return dict(
        task_id=task_id,
        passed=result[0] == "passed",
        result=result[0],
        completion_id=completion_id,
    )


def unsafe_execute(check_program, result, timeout):

    with create_tempdir():

        # These system calls are needed when cleaning up tempdir.
        import os
        import shutil

        rmtree = shutil.rmtree
        rmdir = os.rmdir
        chdir = os.chdir

        # Disable functionalities that can make destructive changes to the test.
        reliability_guard()

        # Run program.
        try:
            exec_globals = {}
            with swallow_io():
                with time_limit(timeout):
                    exec(check_program, exec_globals)
            result.append("passed")
        except TimeoutException:
            result.append("timed out")
        except BaseException as e:
            result.append(f"failed: {e}")

        # Needed for cleaning up.
        shutil.rmtree = rmtree
        os.rmdir = rmdir
        os.chdir = chdir


@contextlib.contextmanager
def time_limit(seconds):
    def signal_handler(signum, frame):
        raise TimeoutException("Timed out!")

    signal.setitimer(signal.ITIMER_REAL, seconds)
    signal.signal(signal.SIGALRM, signal_handler)
    try:
        yield
    finally:
        signal.setitimer(signal.ITIMER_REAL, 0)


@contextlib.contextmanager
def swallow_io():
    stream = WriteOnlyStringIO()
    with contextlib.redirect_stdout(stream):
        with contextlib.redirect_stderr(stream):
            with redirect_stdin(stream):
                yield


@contextlib.contextmanager
def create_tempdir():
    with tempfile.TemporaryDirectory() as dirname:
        with chdir(dirname):
            yield dirname


class TimeoutException(Exception):
    pass


class WriteOnlyStringIO(io.StringIO):
    """StringIO that throws an exception when it's read from"""

    def read(self, *args, **kwargs):
        raise OSError

    def readline(self, *args, **kwargs):
        raise OSError

    def readlines(self, *args, **kwargs):
        raise OSError

    def readable(self, *args, **kwargs):
        """Returns True if the IO object can be read."""
        return False


class redirect_stdin(contextlib._RedirectStream):  # type: ignore
    _stream = "stdin"


@contextlib.contextmanager
def chdir(root):
    if root == ".":
        yield
        return
    cwd = os.getcwd()
    os.chdir(root)
    try:
        yield
    except BaseException as exc:
        raise exc
    finally:
        os.chdir(cwd)


def reliability_guard(maximum_memory_bytes=None):
    """
    This disables various destructive functions and prevents the generated code
    from interfering with the test (e.g. fork bomb, killing other processes,
    removing filesystem files, etc.)

    WARNING
    This function is NOT a security sandbox. Untrusted code, including, model-
    generated code, should not be blindly executed outside of one. See the
    Codex paper for more information about OpenAI's code sandbox, and proceed
    with caution.
    """

    if maximum_memory_bytes is not None:
        import resource

        resource.setrlimit(
            resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)
        )
        resource.setrlimit(
            resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)
        )
        if not platform.uname().system == "Darwin":
            resource.setrlimit(
                resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)
            )

    faulthandler.disable()

    import builtins

    builtins.exit = None
    builtins.quit = None

    import os

    os.environ["OMP_NUM_THREADS"] = "1"

    os.kill = None
    os.system = None
    os.putenv = None
    os.remove = None
    os.removedirs = None
    os.rmdir = None
    os.fchdir = None
    os.setuid = None
    os.fork = None
    os.forkpty = None
    os.killpg = None
    os.rename = None
    os.renames = None
    os.truncate = None
    os.replace = None
    os.unlink = None
    os.fchmod = None
    os.fchown = None
    os.chmod = None
    os.chown = None
    os.chroot = None
    os.fchdir = None
    os.lchflags = None
    os.lchmod = None
    os.lchown = None
    os.getcwd = None
    os.chdir = None

    import shutil

    shutil.rmtree = None
    shutil.move = None
    shutil.chown = None

    import subprocess

    subprocess.Popen = None  # type: ignore

    __builtins__["help"] = None

    import sys

    sys.modules["ipdb"] = None
    sys.modules["joblib"] = None
    sys.modules["resource"] = None
    sys.modules["psutil"] = None
    sys.modules["tkinter"] = None


_WARNING = """
################################################################################
                                  !!!WARNING!!!
################################################################################
The "code_eval" metric executes untrusted model-generated code in Python.
Although it is highly unlikely that model-generated code will do something
overtly malicious in response to this test suite, model-generated code may act
destructively due to a lack of model capability or alignment.
Users are strongly encouraged to sandbox this evaluation suite so that it
does not perform destructive actions on their host or network. For more
information on how OpenAI sandboxes its code, see the paper "Evaluating Large
Language Models Trained on Code" (https://arxiv.org/abs/2107.03374).

Once you have read this disclaimer and taken appropriate precautions,
set the environment variable HF_ALLOW_CODE_EVAL="1". Within Python you can to this
with:

>>> import os
>>> os.environ["HF_ALLOW_CODE_EVAL"] = "1"

################################################################################
"""


def compute(predictions, references, k=[1, 10, 100], num_workers=20, timeout=3.0):
    """Returns the scores"""

    if os.getenv("HF_ALLOW_CODE_EVAL", 0) != "1":
        raise ValueError(_WARNING)

    if os.name == "nt":
        raise NotImplementedError("This metric is currently not supported on Windows.")

    with ThreadPoolExecutor(max_workers=num_workers) as executor:
        futures = []
        completion_id = Counter()
        n_samples = 0
        results = defaultdict(list)

        for task_id, (candidates, test_case) in enumerate(zip(predictions, references)):
            for candidate in candidates:
                test_program = candidate + "\n" + test_case
                args = (test_program, timeout, task_id, completion_id[task_id])
                future = executor.submit(check_correctness, *args)
                futures.append(future)
                completion_id[task_id] += 1
                n_samples += 1

        for future in as_completed(futures):
            result = future.result()
            results[result["task_id"]].append((result["completion_id"], result))

    total, correct = [], []
    for result in results.values():
        result.sort()
        passed = [r[1]["passed"] for r in result]
        total.append(len(passed))
        correct.append(sum(passed))
    total = np.array(total)
    correct = np.array(correct)

    ks = k
    pass_at_k = {
        f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
        for k in ks
        if (total >= k).all()
    }

    return pass_at_k, results


def estimate_pass_at_k(num_samples, num_correct, k):
    """Estimates pass@k of each problem and returns them in an array."""

    def estimator(n: int, c: int, k: int) -> float:
        """Calculates 1 - comb(n - c, k) / comb(n, k)."""
        if n - c < k:
            return 1.0
        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))

    if isinstance(num_samples, int):
        num_samples_it = itertools.repeat(num_samples, len(num_correct))
    else:
        assert len(num_samples) == len(num_correct)
        num_samples_it = iter(num_samples)

    return np.array(
        [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]
    )

evals/elsuite/utils_execute.py
GPT-Fathom/GPT-Fathom
from pytest import mark

from evals.elsuite.utils import fuzzy_match, normalize


@mark.parametrize(
    "s, expected",
    [
        ("", ""),
        ("Hello", "hello"),
        ("hello\nworld", "hello world"),
    ],
)
def test_normalize(s: str, expected: str):
    assert normalize(s) == expected


@mark.parametrize(
    "s1, s2, expected",
    [
        ("", "", True),
        ("x", "", False),
        ("Hello", "Hello", True),
        ("hello", "othello", True),
        ("hello", "oh tello", False),
        ("Hello World", "foo\nhello world", True),
        ("who's there?", "whos there", True),
        ("who's there?", "whosthere", False),
        ("an apple a day that the", "apple day that", True),
    ],
)
def test_fuzzy_match(s1: str, s2: str, expected: bool):
    assert fuzzy_match(s1, s2) == expected
    assert fuzzy_match(s2, s1) == expected

evals/elsuite/utils_test.py
GPT-Fathom/GPT-Fathom
"""
This file defines the base class for evals.
"""
import abc
import asyncio
import logging
import os
import random
from multiprocessing.pool import ThreadPool
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple

from tqdm import tqdm

from evals.api import CompletionFn

from .data import get_jsonl
from .record import RecorderBase
from .registry import Registry

logger = logging.getLogger(__name__)


SHUFFLE_SEED = 123
_MAX_SAMPLES = None


def _index_samples(samples: List[Any]) -> List[Tuple[Any, int]]:
    """Shuffle `samples` and pair each sample with its index."""
    indices = list(range(len(samples)))
    random.Random(SHUFFLE_SEED).shuffle(indices)
    if _MAX_SAMPLES is not None:
        indices = indices[:_MAX_SAMPLES]
    logger.info(f"Evaluating {len(indices)} samples")
    work_items = [(samples[i], i) for i in indices]
    return work_items


def set_max_samples(max_samples: int):
    global _MAX_SAMPLES
    _MAX_SAMPLES = max_samples


class Eval(abc.ABC):
    """
    Evaluation classes generally should override two methods:
    `eval_sample`: Takes in a test sample and a random number generator and
        records the metrics of interest.
    `run`: Takes in a recorder and runs the evaluation. Generally, most `run`
        methods will follow this same pattern: loading the data, calling
        `eval_all_samples`, and aggregating the recorded results.
    """

    def __init__(
        self,
        completion_fns: list[CompletionFn],
        seed: int = 20220722,
        name: str = "no_name_eval.default",
        eval_in_batch: bool = False,
        local_dataset: bool = False,
        registry: Optional[Registry] = None,
        samples_jsonl: Optional[str] = None,
    ):
        splits = name.split(".")
        if len(splits) < 2:
            raise ValueError(
                f"Eval name must at least have <base_eval>.<split>. Got name {name}"
            )

        self.completion_fns = completion_fns
        self.seed = seed
        self.name = name
        self.registry = registry or Registry()
        self.samples_jsonl = samples_jsonl
        self.eval_in_batch = eval_in_batch
        self.local_dataset = local_dataset

    @abc.abstractmethod
    def eval_sample(self, sample: Any, rng: random.Random):
        raise NotImplementedError()

    @property
    def completion_fn(self) -> CompletionFn:
        """Helper for more ergonomic access to a single CompletionFn."""
        return self.completion_fns[0]

    @abc.abstractmethod
    def run(self, recorder: RecorderBase) -> Dict[str, float]:
        """Run the evaluation with the corresponding recorder."""
        raise NotImplementedError()

    async def async_eval_all_samples(
        self,
        eval_fn: Callable[[Tuple[Any, int]], Awaitable[Tuple[int, Any]]],
        samples: List[Any],
        concurrency: int = 32,
        show_progress: bool = True,
        **_kwargs: Any,
    ):
        work_items = _index_samples(samples)
        semaphore = asyncio.Semaphore(concurrency)

        async def eval_fn_with_semaphore(args):
            async with semaphore:
                return await eval_fn(args)

        futures = [
            asyncio.ensure_future(eval_fn_with_semaphore(args)) for args in work_items
        ]

        for future in tqdm(
            asyncio.as_completed(futures), total=len(samples), disable=not show_progress
        ):
            await future

    def eval_all_samples(
        self,
        recorder: RecorderBase,
        samples,
        show_progress=True,
        record_raw_sample=True,
        **_kwargs: Any,
    ):
        """
        Evaluate all provided samples in parallel.
        """
        work_items = _index_samples(samples)
        threads = int(os.environ.get("EVALS_THREADS", "10"))
        show_progress = bool(os.environ.get("EVALS_SHOW_EVAL_PROGRESS", show_progress))

        def eval_sample(args):
            """
            Evaluate a single sample.
            """
            sample, idx = args
            base_name, split = self.name.split(".")[0:2]
            sample_id = f"{base_name}.{split}.{idx}"
            with recorder.as_default_recorder(sample_id):
                seed = f"{sample_id}:{self.seed}".encode("utf-8")
                rng = random.Random(seed)
                return idx, self.eval_sample(sample, rng)

        with ThreadPool(threads) as pool:
            if os.environ.get("EVALS_SEQUENTIAL", "0") in {"1", "true", "yes"}:
                logger.info(f"Running in sequential mode!")
                iter = map(eval_sample, work_items)
            else:
                logger.info(f"Running in threaded mode with {threads} threads!")
                iter = pool.imap_unordered(eval_sample, work_items)
            idx_and_result = list(
                tqdm(iter, total=len(work_items), disable=not show_progress)
            )
        return [r for _, r in sorted(idx_and_result)]

    def get_samples(self):
        if self.samples_jsonl is None:
            raise ValueError(
                "To use `get_samples`, you must provide a `samples_jsonl` path."
                "Got `None`."
            )

        return get_jsonl(self.samples_jsonl)

evals/eval.py
GPT-Fathom/GPT-Fathom
"""
This file defines utilities for adding multiple choice questions to prompts.
"""
import random
from typing import Optional


def make_abc(
    answers, *, correct_idx=0, shuffle=True, rng: Optional[random.Random] = None
):
    """
    ARGS
    ====
    `answers`: A sequence of strings, each of which is an answer choice.
    `correct_idx`: The integer index of the correct answer.
    `shuffle`: If True, shuffle the answer choices in the returned string.
    `rng`: If `shuffle` is True, this is the random number generator to use.

    RETURNS
    =======
    A tuple of (options, correct_answer) where `options` is a string of
    newline-separated answer choices (e.g., "A) blah") and `correct_answer` is
    the correct answer as a single character (e.g., "A").
    """

    p = list(range(len(answers)))

    if shuffle:
        if rng is None:
            raise ValueError("shuffle=True requires rng")
        rng.shuffle(p)

    options = ""
    for i, j in enumerate(p):
        if i > 0:
            options += "\n"
        options += "(" + chr(ord("A") + i) + ") " + answers[j]
    return (
        options,
        chr(ord("A") + p.index(correct_idx)),
        "(" + chr(ord("A") + p.index(correct_idx)) + ") " + answers[correct_idx],
    )

evals/formatting.py
GPT-Fathom/GPT-Fathom
"""
This file defines various common metrics of interest.
"""
import random
from typing import Optional, Sequence, Set

import numpy as np

from evals.record import Event


def get_accuracy(events: Sequence[Event]) -> float:
    num_correct = sum(int(event.data["correct"]) for event in events)
    num_total = len(events)
    if num_total == 0:
        return float("nan")
    else:
        return num_correct / num_total


def get_picked(events: Sequence[Event]) -> float:
    num_correct = sum(int(event.data["detected"]) for event in events)
    num_total = len(events)
    if num_total == 0:
        return float("nan")
    else:
        return num_correct / num_total


def get_bootstrap_accuracy_std(
    events: Sequence[Event], num_samples: int = 1000
) -> float:
    vals = [m.data["correct"] for m in events]
    return np.std(
        [np.mean(random.sample(vals, len(vals) // 2)) for _ in range(num_samples)]
    )


def get_confusion_matrix(
    matches: Sequence[Event], class_labels: Optional[Set] = None
) -> np.ndarray:
    labels = {match.data["expected"] for match in matches}
    if class_labels is None:
        labels = {label: i for i, label in enumerate(sorted(labels))}
    else:
        assert labels.issubset(class_labels)
        labels = {label: i for i, label in enumerate(class_labels)}
    result = np.zeros((len(labels), len(labels) + 1), dtype=int)
    for match in matches:
        i = labels[match.data["expected"]]
        j = labels.get(match.data["picked"], len(labels))
        result[i, j] += 1
    return result


def compute_matthew_corr(confusion_matrix: np.ndarray) -> float:
    assert confusion_matrix.shape == (2, 3), f"Got shape: {confusion_matrix.shape}"
    r = confusion_matrix[:, :2]
    r[:, 0] += confusion_matrix[:, 2]
    return (r[1, 1] * r[0, 0] - r[1, 0] * r[0, 1]) / np.sqrt(
        r[1, :].sum() * r[0, :].sum() * r[:, 0].sum() * r[:, 1].sum()
    )


def compute_precision(confusion_matrix: np.ndarray, idx: int = 0) -> float:
    return confusion_matrix[idx, idx] / confusion_matrix[:, idx].sum()


def compute_recall(confusion_matrix: np.ndarray, idx: int = 0) -> float:
    return confusion_matrix[idx, idx] / confusion_matrix[idx, :].sum()


def compute_f_score(
    confusion_matrix: np.ndarray, idx: int = 0, beta: float = 1.0
) -> float:
    precision = compute_precision(confusion_matrix, idx=idx)
    recall = compute_recall(confusion_matrix, idx=idx)
    return (1 + beta**2) * (precision * recall) / (beta**2 * precision + recall)


def compute_averaged_f_score(
    confusion_matrix: np.ndarray, beta: float = 1.0, average: str = "macro"
) -> float:
    assert average in ["macro"]
    f_scores = []
    for i in range(confusion_matrix.shape[0]):
        f_scores.append(compute_f_score(confusion_matrix, idx=i, beta=beta))
    return np.array(f_scores).mean()

evals/metrics.py
GPT-Fathom/GPT-Fathom
"""
This file defines the classes for how to manage prompts for different types of
models, i.e., "chat models" vs. "non chat models".
"""
import logging
import threading
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Dict, List, Union

logger = logging.getLogger(__name__)
ENCODER_LOCK = threading.Lock()

# This is an approximation to the type accepted as the `prompt` field to `openai.Completion.create` calls
OpenAICreatePrompt = Union[str, list[str], list[int], list[list[int]]]

# This is the type accepted as the `prompt` field to `openai.ChatCompletion.create` calls
OpenAIChatMessage = Dict[
    str, str
]  # A message is a dictionary with "role" and "content" keys
OpenAICreateChatPrompt = List[OpenAIChatMessage]  # A chat log is a list of messages


def chat_prompt_to_text_prompt(
    prompt: OpenAICreateChatPrompt,
    Q_A_ENDING: bool = False,
    new_line_end: bool = False,
    for_completion: bool = True,
) -> str:
    """
    Render a chat prompt as a text prompt. User and assistant messages are separated by newlines
    and prefixed with "User: " and "Assistant: ", respectively, unless there is only one message.
    System messages have no prefix.
    """
    assert is_chat_prompt(prompt), f"Expected a chat prompt, got {prompt}"
    chat_to_prefixes = {
        # roles
        "system": "",
        # names
        "example_user": "Q: ",
        "example_assistant": "A: ",
        "user": "Q: " if Q_A_ENDING else "",
    }

    # For a single message, be it system, user, or assistant, just return the message
    if len(prompt) == 1:
        return prompt[0]["content"]

    text = ""
    few_shot = 0
    for msg in prompt:
        role = msg["name"] if "name" in msg else msg["role"]
        if role == "example_user":
            few_shot = 1
        prefix = chat_to_prefixes.get(role, role.capitalize() + ": ")
        content = msg["content"]
        text += f"{prefix}{content}\n"
    if Q_A_ENDING:
        text += "A: "
    if new_line_end:
        if text[-1] != "\n":
            text += "\n"
    else:
        if text[-1] == "\n":
            text = text[:-1]
    # if few_shot == 0:
    #     text = text[:-1]
    # text = text[:-1]
    return text.lstrip()


def text_prompt_to_chat_prompt(
    prompt: str, role: str = "system"
) -> OpenAICreateChatPrompt:
    assert isinstance(prompt, str), f"Expected a text prompt, got {prompt}"

    return [
        {
            "role": "system",
            "content": "Follow the given instructions and examples, answer the question.",
        },
        {"role": "user", "content": prompt},
    ]


@dataclass
class Prompt(ABC):
    """
    A `Prompt` encapsulates everything required to present the `raw_prompt` in different formats,
    e.g., a normal unadorned format vs. a chat format.
    """

    @abstractmethod
    def to_formatted_prompt(self):
        """
        Return the actual data to be passed as the `prompt` field to your model.
        See the above types to see what each API call is able to handle.
        """


def is_chat_prompt(prompt: Prompt) -> bool:
    return isinstance(prompt, list) and all(isinstance(msg, dict) for msg in prompt)


@dataclass
class CompletionPrompt(Prompt):
    """
    A `Prompt` object that wraps prompts to be compatible with non chat models, which use `openai.Completion.create`.
    """

    raw_prompt: Union[str, OpenAICreateChatPrompt]

    def _render_chat_prompt_as_text(self, prompt: OpenAICreateChatPrompt) -> str:
        return chat_prompt_to_text_prompt(prompt)

    def to_formatted_prompt(self) -> str:
        if is_chat_prompt(self.raw_prompt):
            return self._render_chat_prompt_as_text(self.raw_prompt)
        return self.raw_prompt


@dataclass
class ChatCompletionPrompt(Prompt):
    """
    A `Prompt` object that wraps prompts to be compatible with chat models, which use `openai.ChatCompletion.create`.

    The format expected by chat models is a list of messages, where each message is a dict with "role" and "content" keys.
    """

    raw_prompt: Union[OpenAICreatePrompt, OpenAICreateChatPrompt]

    def _render_text_as_chat_prompt(self, prompt: str) -> OpenAICreateChatPrompt:
        """
        Render a text string as a chat prompt. The default option we adopt here is to simply take the full prompt
        and treat it as a system message.
        """
        return text_prompt_to_chat_prompt(prompt)

    def to_formatted_prompt(self) -> OpenAICreateChatPrompt:
        if is_chat_prompt(self.raw_prompt):
            return self.raw_prompt
        return self._render_text_as_chat_prompt(self.raw_prompt)

evals/prompt/base.py
GPT-Fathom/GPT-Fathom
from evals.prompt.base import chat_prompt_to_text_prompt, text_prompt_to_chat_prompt

prompt1 = [
    {
        "role": "system",
        "content": "Follow the given instructions and examples, answer the question.",
    },
    {"role": "user", "content": "Hello World!"},
]

text_prompt1 = chat_prompt_to_text_prompt(prompt1)
print(text_prompt1)


prompt2 = [
    {
        "role": "system",
        "content": "Follow the given instructions and examples, answer the question.",
    },
    {"role": "system", "name": "example_user", "content": "1 + 1 = "},
    {"role": "system", "name": "example_assistant", "content": "12"},
    {"role": "system", "content": "predict the next word"},
    {"role": "user", "content": "Hello World!"},
]
text_prompt2 = chat_prompt_to_text_prompt(prompt2)
print(text_prompt2)

evals/prompt/base_test.py
GPT-Fathom/GPT-Fathom
"""
This file defines the recorder classes which log eval results in different ways,
such as to a local JSON file or to a remote Snowflake database.

If you would like to implement a custom recorder, you can see how the
`LocalRecorder` and `Recorder` classes inherit from the `RecorderBase` class and
override certain methods.
"""
import atexit
import contextlib
import dataclasses
import logging
import os
import threading
import time
from contextvars import ContextVar
from datetime import datetime, timezone
from typing import Any, List, Optional, Sequence

import blobfile as bf

import evals
from evals.base import RunSpec
from evals.data import jsondumps
from evals.utils.misc import t
from evals.utils.snowflake import SnowflakeConnection

logger = logging.getLogger(__name__)

MIN_FLUSH_EVENTS = 100
MAX_SNOWFLAKE_BYTES = 16 * 10**6
MIN_FLUSH_SECONDS = 10

_default_recorder: ContextVar[Optional["RecorderBase"]] = ContextVar(
    "default_recorder", default=None
)


def default_recorder() -> Optional["RecorderBase"]:
    return _default_recorder.get()


@dataclasses.dataclass
class Event:
    run_id: str
    event_id: int
    sample_id: Optional[str]
    type: str
    data: dict
    created_by: str
    created_at: str


class RecorderBase:
    """
    The standard events for which recording methods are provided are:
    - `match`: A match or non match, as specified by the `correct` bool, between
        the `expected` and `picked` results.
    - `embedding`: An embedding of the `prompt` of type `embedding_type`.
    - `sampling`: What was `sampled` from the model given the input `prompt`.
    - `cond_logp`: The conditional log probability, as `logp`, of the
        `completion` from the model given the input `prompt`.
    - `pick_option`: The option `picked` by the model out of the valid `options`
        given the input `prompt`.
    - `raw`: A raw sample specified by the `data`.
    - `metrics`: A set of metrics specified by the `kwargs`.
    - `error`: An `error` along with an accompanying `msg`.
    - `extra`: Any extra `data` of interest to be recorded.
    For these events, helper methods are defined at the bottom of this file.
    More generally, you can record any event by calling `record_event` with the
    event `type` and `data`.
    Finally, you can also record a final report using `record_final_report`.
    """

    def __init__(
        self,
        run_spec: evals.base.RunSpec,
    ) -> None:
        self._sample_id: ContextVar[Optional[int]] = ContextVar(
            "_sample_id", default=None
        )
        self.run_spec = run_spec
        self._events: List[Event] = []
        self._last_flush_time = time.time()
        self._flushes_done = 0
        self._written_events = 0
        self._flushes_started = 0
        self._event_lock = threading.Lock()
        self._paused_ids: List[str] = []
        atexit.register(self.flush_events)

    @contextlib.contextmanager
    def as_default_recorder(self, sample_id: str):
        sample_id_token = self._sample_id.set(sample_id)
        default_recorder_token = _default_recorder.set(self)
        yield
        _default_recorder.reset(default_recorder_token)
        self._sample_id.reset(sample_id_token)

    def current_sample_id(self) -> Optional[str]:
        return self._sample_id.get()

    def pause(self):
        sample_id = self.current_sample_id()
        with self._event_lock:
            if sample_id not in self._paused_ids:
                self._paused_ids.append(sample_id)

    def unpause(self):
        sample_id = self.current_sample_id()
        with self._event_lock:
            if sample_id in self._paused_ids:
                self._paused_ids.remove(sample_id)

    def is_paused(self, sample_id: str = None):
        if sample_id is None:
            sample_id = self.current_sample_id()
        with self._event_lock:
            return sample_id in self._paused_ids

    def get_events(self, type: str) -> Sequence[Event]:
        with self._event_lock:
            return [event for event in self._events if event.type == type]

    def get_metrics(self):
        return list(map(lambda x: x.data, self.get_events("metrics")))

    def get_scores(self, key: str):
        return list(map(lambda e: e.data[key], self.get_events("metrics")))

    def _create_event(self, type, data=None, sample_id=None):
        if sample_id is None:
            sample_id = self.current_sample_id()
        if sample_id is None:
            raise ValueError(
                "No sample_id set! Either pass it in or use as_default_recorder!"
            )

        return Event(
            run_id=self.run_spec.run_id,
            event_id=len(self._events),
            type=type,
            sample_id=sample_id,
            data=data,
            created_by=self.run_spec.created_by,
            created_at=str(datetime.now(timezone.utc)),
        )

    def _flush_events_internal(self, events_to_write: Sequence[Event]):
        pass

    def flush_events(self):
        with self._event_lock:
            if len(self._events) == self._written_events:
                return
            events_to_write = self._events[self._written_events :]
            self._written_events = len(self._events)
            self._flushes_started += 1
        self._flush_events_internal(events_to_write)

    def record_event(self, type, data=None, sample_id=None):
        if sample_id is None:
            sample_id = self.current_sample_id()
        if sample_id is None:
            raise ValueError(
                "No sample_id set! Either pass it in or use as_default_recorder!"
            )

        if self.is_paused(sample_id):
            return
        with self._event_lock:
            event = Event(
                run_id=self.run_spec.run_id,
                event_id=len(self._events),
                type=type,
                sample_id=sample_id,
                data=data,
                created_by=self.run_spec.created_by,
                created_at=str(datetime.now(timezone.utc)),
            )
            self._events.append(event)
            if (
                self._flushes_done < self._flushes_started
                or len(self._events) < self._written_events + MIN_FLUSH_EVENTS
                or time.time() < self._last_flush_time + MIN_FLUSH_SECONDS
            ):
                return
            events_to_write = self._events[self._written_events :]
            self._written_events = len(self._events)
            self._flushes_started += 1
            self._flush_events_internal(events_to_write)

    def record_match(
        self,
        correct: bool,
        *,
        expected=None,
        picked=None,
        sample_id=None,
        detected=False,
        **extra,
    ):
        assert isinstance(
            correct, bool
        ), f"correct must be a bool, but was a {type(correct)}: {correct}"

        if isinstance(expected, list) and len(expected) == 1:
            expected = expected[0]
        data = {
            "correct": bool(correct),
            "expected": expected,
            "picked": picked,
            "detected": detected,
            **extra,
        }
        self.record_event("match", data, sample_id=sample_id)

    def record_embedding(self, prompt, embedding_type, sample_id=None, **extra):
        data = {
            "prompt": prompt,
            "embedding_type": embedding_type,
            **extra,
        }
        self.record_event("embedding", data, sample_id=sample_id)

    def record_sampling(self, prompt, sampled, sample_id=None, **extra):
        data = {
            "prompt": prompt,
            "sampled": sampled,
            **extra,
        }
        self.record_event("sampling", data, sample_id=sample_id)

    def record_cond_logp(self, prompt, completion, logp, sample_id=None, **extra):
        data = {
            "prompt": prompt,
            "completion": completion,
            "logp": logp,
            **extra,
        }
        self.record_event("cond_logp", data, sample_id=sample_id)

    def record_pick_option(self, prompt, options, picked, sample_id=None, **extra):
        data = {
            "prompt": prompt,
            "options": options,
            "picked": picked,
            **extra,
        }
        self.record_event("pick_option", data, sample_id=sample_id)

    def record_raw(self, data):
        self.record_event("raw_sample", data)

    def record_metrics(self, **kwargs):
        self.record_event("metrics", kwargs)

    def record_error(self, msg: str, error: Exception, **kwargs):
        data = {
            "type": type(error).__name__,
            "message": str(error),
        }
        data.update(kwargs)
        self.record_event("error", data)

    def record_extra(self, data, sample_id=None):
        self.record_event("extra", data, sample_id=sample_id)

    def record_final_report(self, final_report: Any):
        logging.info(f"Final report: {final_report}. Not writing anywhere.")


def _green(str):
    return f"\033[1;32m{str}\033[0m"


def _red(str):
    return f"\033[1;31m{str}\033[0m"


class DummyRecorder(RecorderBase):
    """
    A "recorder" which only logs certain events to the console.
    Can be used by passing `--dry-run` when invoking `oaieval`.
    """

    def __init__(self, run_spec: RunSpec, log: bool = True):
        super().__init__(run_spec)
        self.log = log

    def record_event(self, type, data, sample_id=None):
        from evals.registry import registry

        if self.run_spec is None:
            return

        base_eval_spec = registry.get_base_eval(self.run_spec.base_eval)
        if base_eval_spec and len(base_eval_spec.metrics) >= 1:
            primary_metric = base_eval_spec.metrics[0]
        else:
            primary_metric = "accuracy"

        with self._event_lock:
            event = self._create_event(type, data)
            self._events.append(event)

        msg = f"Not recording event: {event}"

        if type == "match":
            accuracy_good = (
                primary_metric == "accuracy" or primary_metric.startswith("pass@")
            ) and (data.get("correct", False) or data.get("accuracy", 0) > 0.5)
            f1_score_good = (
                primary_metric == "f1_score" and data.get("f1_score", 0) > 0.5
            )
            if accuracy_good or f1_score_good:
                msg = _green(msg)
            else:
                msg = _red(msg)

        if self.log:
            logging.info(msg)


class LocalRecorder(RecorderBase):
    """
    A recorder which logs events to the specified JSON file.
    This is the default recorder used by `oaieval`.
    """

    def __init__(self, log_path: Optional[str], run_spec: RunSpec):
        super().__init__(run_spec)
        self.event_file_path = log_path
        if log_path is not None:
            with bf.BlobFile(log_path, "wb") as f:
                f.write(
                    (jsondumps({"spec": dataclasses.asdict(run_spec)}) + "\n").encode(
                        "utf-8"
                    )
                )

    def _flush_events_internal(self, events_to_write: Sequence[Event]):
        start = time.time()
        try:
            lines = [jsondumps(event) + "\n" for event in events_to_write]
        except TypeError as e:
            logger.error(f"Failed to serialize events: {events_to_write}")
            raise e

        with bf.BlobFile(self.event_file_path, "ab") as f:
            f.write(b"".join([l.encode("utf-8") for l in lines]))

        logger.info(
            f"Logged {len(lines)} rows of events to {self.event_file_path}: insert_time={t(time.time()-start)}"
        )

        self._last_flush_time = time.time()
        self._flushes_done += 1

    def record_final_report(self, final_report: Any):
        with bf.BlobFile(self.event_file_path, "ab") as f:
            f.write((jsondumps({"final_report": final_report}) + "\n").encode("utf-8"))

        logging.info(f"Final report: {final_report}. Logged to {self.event_file_path}")
        print(
            f"--- {self.run_spec.setting_name}  Final report: {final_report}. Logged to {self.event_file_path}"
        )

        filename = f"/tmp/res/{self.run_spec.completion_fns[0]}.json"
        print(f"--- Score has been updated in the JSON file for {self.run_spec.completion_fns[0]}.")
        print(f"File location: {filename}")
        import json
        from pathlib import Path

        path = f"/tmp/res/{self.run_spec.completion_fns[0]}.json"
        p = Path(path)

        if os.path.exists(path):
            with open(path, "r") as file:
                loaded_data = json.load(file)
        else:
            p.parent.mkdir(parents=True, exist_ok=True)
            loaded_data = {}
        loaded_data[self.run_spec.setting_name] = (final_report, self.event_file_path)
        with open(path, "w") as file:
            json.dump(loaded_data, file)


class Recorder(RecorderBase):
    """
    A recorder which logs events to Snowflake.
    Can be used by passing `--no-local-run` when invoking `oaieval`.
    """

    def __init__(
        self,
        log_path: Optional[str],
        run_spec: RunSpec,
        snowflake_connection: Optional[SnowflakeConnection] = None,
    ) -> None:
        super().__init__(run_spec)
        self.event_file_path = log_path
        self._writing_lock = threading.Lock()

        if snowflake_connection is None:
            snowflake_connection = SnowflakeConnection()
        self._conn = snowflake_connection

        if log_path is not None:
            with bf.BlobFile(log_path, "wb") as f:
                f.write(
                    (jsondumps({"spec": dataclasses.asdict(run_spec)}) + "\n").encode(
                        "utf-8"
                    )
                )

        query = """
            INSERT ALL INTO runs (run_id, model_name, eval_name, base_eval, split, run_config, settings, created_by, created_at)
            VALUES (%(run_id)s, %(model_name)s, %(eval_name)s, %(base_eval)s, %(split)s, run_config, settings, %(created_by)s, %(created_at)s)
            SELECT PARSE_JSON(%(run_config)s) AS run_config, PARSE_JSON(%(settings)s) AS settings
        """
        self._conn.robust_query(
            command=query,
            params={
                "run_id": run_spec.run_id,
                # TODO: model_name -> completion_fns
                "model_name": jsondumps(dict(completions=run_spec.completion_fns)),
                "eval_name": run_spec.eval_name,
                "base_eval": run_spec.base_eval,
                "split": run_spec.split,
                "run_config": jsondumps(run_spec.run_config),
                "settings": jsondumps(run_spec.run_config.get("initial_settings", {})),
                "created_by": run_spec.created_by,
                "created_at": run_spec.created_at,
            },
        )
        atexit.register(self.flush_events)

    def _flush_events_internal(self, events_to_write: Sequence[Event]):
        with self._writing_lock:
            try:
                lines = [jsondumps(event) + "\n" for event in events_to_write]
            except TypeError as e:
                logger.error(f"Failed to serialize events: {events_to_write}")
                raise e
            idx_l = 0
            while idx_l < len(events_to_write):
                total_bytes = 0
                idx_r = idx_l
                while (
                    idx_r < len(events_to_write)
                    and total_bytes + len(lines[idx_r]) < MAX_SNOWFLAKE_BYTES
                ):
                    total_bytes += len(lines[idx_r])
                    idx_r += 1
                assert idx_r > idx_l
                start = time.time()
                buffer = [
                    (
                        event.run_id,
                        event.event_id,
                        event.sample_id,
                        event.type,
                        jsondumps(event.data),
                        event.created_by,
                        event.created_at,
                    )
                    for event in events_to_write[idx_l:idx_r]
                ]
                query = """
                INSERT INTO events (run_id, event_id, sample_id, type, data, created_by, created_at)
                SELECT Column1 AS run_id, Column2 as event_id, Column3 AS sample_id, Column4 AS type, PARSE_JSON(Column5) AS data, Column6 AS created_by, Column7 AS created_at
                FROM VALUES(%s, %s, %s, %s, %s, %s, %s)
                """
                self._conn.robust_query(command=query, seqparams=buffer, many=True)
                logger.info(
                    f"Logged {len(buffer)} rows of events to Snowflake: insert_time={t(time.time()-start)}"
                )
                idx_l = idx_r

            with bf.BlobFile(self.event_file_path, "ab") as f:
                f.write(b"".join([l.encode("utf-8") for l in lines]))
            self._last_flush_time = time.time()
            self._flushes_done += 1

    def record_final_report(self, final_report: Any):
        with self._writing_lock:
            with bf.BlobFile(self.event_file_path, "ab") as f:
                f.write(
                    (jsondumps({"final_report": final_report}) + "\n").encode("utf-8")
                )
            query = """
                UPDATE runs
                SET final_report = PARSE_JSON(%(final_report)s)
                WHERE run_id = %(run_id)s
            """
            self._conn.robust_query(
                command=query,
                params={
                    "run_id": self.run_spec.run_id,
                    "final_report": jsondumps(final_report),
                },
            )

    def record_event(self, type, data=None, sample_id=None):
        # try to serialize data so we fail early!
        _ = jsondumps(data)
        return super().record_event(type, data, sample_id)


#########################################################################
### Helper methods which use the thread local global default recorder ###
#########################################################################


def current_sample_id() -> str:
    return default_recorder().current_sample_id


def record_match(correct: bool, *, expected=None, picked=None, **extra):
    return default_recorder().record_match(
        correct, expected=expected, picked=picked, **extra
    )


def record_embedding(prompt, embedding_type, **extra):
    return default_recorder().record_embedding(prompt, embedding_type, **extra)


def record_sampling(prompt, sampled, **extra):
    return default_recorder().record_sampling(prompt, sampled, **extra)


def record_cond_logp(prompt, completion, logp, **extra):
    return default_recorder().record_cond_logp(prompt, completion, logp, **extra)


def record_pick_option(prompt, options, picked, **extra):
    return default_recorder().record_pick_option(prompt, options, picked, **extra)


def record_raw(data):
    return default_recorder().record_raw(data)


def record_metrics(**extra):
    return default_recorder().record_metrics(**extra)


def record_error(msg: str, error: Exception = None, **extra):
    return default_recorder().record_error(msg, error, **extra)


def record_extra(data):
    return default_recorder().record_extra(data)


def record_event(type, data=None, sample_id=None):
    return default_recorder().record_event(type, data, sample_id)


def pause():
    return default_recorder().pause()


def unpause():
    return default_recorder().unpause()

evals/record.py
GPT-Fathom/GPT-Fathom
"""
Functions to handle registration of evals. To add a new eval to the registry,
add an entry in one of the YAML files in the `../registry` dir.
By convention, every eval name should start with {base_eval}.{split}.
"""

import copy
import difflib
import functools
import logging
import os
import re
from functools import cached_property
from pathlib import Path
from typing import Any, Iterator, Optional, Sequence, Type, TypeVar, Union

import openai
import yaml

from evals import (
    AzureChatCompletionFn,
    AzureCompletionFn,
    OpenAIChatCompletionFn,
    OpenAICompletionFn,
)
from evals.api import CompletionFn, DummyCompletionFn
from evals.base import BaseEvalSpec, CompletionFnSpec, EvalSetSpec, EvalSpec
from evals.elsuite.modelgraded.base import ModelGradedSpec
from evals.utils.misc import make_object

logger = logging.getLogger(__name__)

DEFAULT_PATHS = [
    Path(__file__).parents[0].resolve() / "registry",
    Path.home() / ".evals",
]


def n_ctx_from_model_name(model_name: str) -> Optional[int]:
    """Returns n_ctx for a given API model name. Model list last updated 2023-03-14."""
    # note that for most models, the max tokens is n_ctx + 1
    DICT_OF_N_CTX_BY_MODEL_NAME_PREFIX: dict[str, int] = {
        "gpt-3.5": 4096,
        "gpt-4": 8192,
        "gpt-4-32k-": 32768,
    }
    DICT_OF_N_CTX_BY_MODEL_NAME: dict[str, int] = {
        "ada": 2048,
        "text-ada-001": 2048,
        "babbage": 2048,
        "text-babbage-001": 2048,
        "code-cushman-001": 2048,
        "curie": 2048,
        "llama": 2048,
        "llama2": 4096,
        "text-curie-001": 2048,
        "davinci": 2048,
        "text-davinci-001": 2048,
        "davinci-instruct-beta": 2048,
        "code-davinci-002": 8000,
        "text-davinci-002": 4096,
        "text-davinci-003": 4096,
        "gpt-3.5-turbo": 4096,
        "gpt-3.5-turbo-0301": 4096,
        "gpt-4": 8192,
        "gpt-4-0314": 8192,
        "gpt-4-32k": 32768,
        "gpt-4-32k-0314": 32768,
    }
    # first, look for a prefix match
    for model_prefix, n_ctx in DICT_OF_N_CTX_BY_MODEL_NAME_PREFIX.items():
        if model_name.startswith(model_prefix):
            return n_ctx
    # otherwise, look for an exact match and return None if not found
    return DICT_OF_N_CTX_BY_MODEL_NAME.get(model_name, None)


CHAT_MODELS = {
    "gpt-3.5",
    "gpt-3.5-frontend",
    "gpt-3.5-turbo",
    "gpt-3.5-turbo-0301",
    "gpt-3.5-turbo-0613",
    "gpt-4",
    "gpt-4-frontend",
    "gpt-4-code-interpreter-frontend",
    "gpt-4-0314",
    "gpt-4-32k",
    "gpt-4-0613",
    "gpt-4-32k-0314",
}


T = TypeVar("T")
RawRegistry = dict[str, Any]


class Registry:
    def __init__(self, registry_paths: Sequence[Union[str, Path]] = DEFAULT_PATHS):
        self._registry_paths = [
            Path(p) if isinstance(p, str) else p for p in registry_paths
        ]

    def add_registry_paths(self, paths: list[Union[str, Path]]) -> None:
        self._registry_paths.extend(
            [Path(p) if isinstance(p, str) else p for p in paths]
        )

    @cached_property
    def api_model_ids(self) -> list[str]:
        try:
            return [m["id"] for m in openai.Model.list()["data"]]
        except openai.error.OpenAIError as err:
            # Errors can happen when running eval with completion function that uses custom
            # API endpoints and authentication mechanisms.
            logger.warning(f"Could not fetch API model IDs from OpenAI API: {err}")
            return []

    def make_completion_fn(self, azure_eval: bool, name: str) -> CompletionFn:
        """
        Create a CompletionFn. The name can be one of the following formats:
        1. openai-model-id (e.g. "gpt-3.5-turbo")
        2. completion-fn-id (from the registry)
        """

        if name == "dummy":
            return DummyCompletionFn()

        n_ctx = n_ctx_from_model_name(name)

        if azure_eval == False:
            if name in CHAT_MODELS:
                return OpenAIChatCompletionFn(model=name, n_ctx=n_ctx)
            elif name in self.api_model_ids:
                return OpenAICompletionFn(model=name, n_ctx=n_ctx)
        else:
            if name in CHAT_MODELS:
                return AzureChatCompletionFn(model=name, n_ctx=n_ctx)
            else:
                return AzureCompletionFn(model=name, n_ctx=n_ctx)

        # No match, so try to find a completion-fn-id in the registry
        spec = self.get_completion_fn(name)
        if spec is None:
            raise ValueError(
                f"Could not find CompletionFn in the registry with ID {name}"
            )
        if spec.args is None:
            spec.args = {}

        spec.args["registry"] = self
        instance = make_object(spec.cls)(**spec.args or {})
        assert isinstance(instance, CompletionFn), f"{name} must be a CompletionFn"
        return instance

    def get_class(self, spec: EvalSpec) -> Any:
        return make_object(spec.cls, **(spec.args if spec.args else {}))

    def _dereference(
        self, name: str, d: RawRegistry, object: str, type: Type[T], **kwargs: dict
    ) -> Optional[T]:
        if not name in d:
            logger.warning(
                (
                    f"{object} '{name}' not found. "
                    f"Closest matches: {difflib.get_close_matches(name, d.keys(), n=5)}"
                )
            )
            return None

        def get_alias() -> Optional[str]:
            if isinstance(d[name], str):
                return d[name]
            if isinstance(d[name], dict) and "id" in d[name]:
                return d[name]["id"]
            return None

        logger.debug(f"Looking for {name}")
        while True:
            alias = get_alias()

            if alias is None:
                break
            name = alias

        spec = d[name]
        if kwargs:
            spec = copy.deepcopy(spec)
            spec.update(kwargs)

        try:
            return type(**spec)
        except TypeError as e:
            raise TypeError(f"Error while processing {object} '{name}': {e}")

    def get_modelgraded_spec(
        self, name: str, **kwargs: dict
    ) -> Optional[ModelGradedSpec]:
        assert name in self._modelgraded_specs, (
            f"Modelgraded spec {name} not found. "
            f"Closest matches: {difflib.get_close_matches(name, self._modelgraded_specs.keys(), n=5)}"
        )
        return self._dereference(
            name, self._modelgraded_specs, "modelgraded spec", ModelGradedSpec, **kwargs
        )

    def get_completion_fn(self, name: str) -> Optional[CompletionFnSpec]:
        return self._dereference(
            name, self._completion_fns, "completion_fn", CompletionFnSpec
        )

    def get_eval(self, name: str) -> Optional[EvalSpec]:
        return self._dereference(name, self._evals, "eval", EvalSpec)

    def get_eval_set(self, name: str) -> Optional[EvalSetSpec]:
        return self._dereference(name, self._eval_sets, "eval set", EvalSetSpec)

    def get_evals(self, patterns: Sequence[str]) -> Iterator[Optional[EvalSpec]]:
        # valid patterns: hello, hello.dev*, hello.dev.*-v1
        def get_regexp(pattern: str) -> re.Pattern[str]:
            pattern = pattern.replace(".", "\\.")
            pattern = pattern.replace("*", ".*")
            return re.compile(f"^{pattern}$")

        regexps = list(map(get_regexp, patterns))
        for name in self._evals:
            # if any regexps match, return the name
            if any(map(lambda regexp: regexp.match(name), regexps)):
                yield self.get_eval(name)

    def get_base_evals(self) -> list[Optional[BaseEvalSpec]]:
        base_evals: list[Optional[BaseEvalSpec]] = []
        for name, spec in self._evals.items():
            if name.count(".") == 0:
                base_evals.append(self.get_base_eval(name))
        return base_evals

    def get_base_eval(self, name: str) -> Optional[BaseEvalSpec]:
        if not name in self._evals:
            return None

        spec_or_alias = self._evals[name]
        if isinstance(spec_or_alias, dict):
            spec = spec_or_alias
            try:
                return BaseEvalSpec(**spec)
            except TypeError as e:
                raise TypeError(f"Error while processing base eval {name}: {e}")

        alias = spec_or_alias
        return BaseEvalSpec(id=alias)

    def _process_file(self, registry: RawRegistry, path: Path) -> None:
        with open(path, "r", encoding="utf-8") as f:
            d = yaml.safe_load(f)

        if d is None or not isinstance(d, dict):
            # no entries in the file
            return

        for name, spec in d.items():
            assert name not in registry, f"duplicate entry: {name} from {path}"
            if isinstance(spec, dict):
                if "key" in spec:
                    raise ValueError(
                        f"key is a reserved keyword, but was used in {name} from {path}"
                    )
                if "group" in spec:
                    raise ValueError(
                        f"group is a reserved keyword, but was used in {name} from {path}"
                    )
                if "cls" in spec:
                    raise ValueError(
                        f"cls is a reserved keyword, but was used in {name} from {path}"
                    )

                spec["key"] = name
                spec["group"] = str(os.path.basename(path).split(".")[0])
                if "class" in spec:
                    spec["cls"] = spec["class"]
                    del spec["class"]
            registry[name] = spec

    def _process_directory(self, registry: RawRegistry, path: Path) -> None:
        files = Path(path).glob("*.yaml")
        for file in files:
            self._process_file(registry, file)

    def _load_registry(self, paths: Sequence[Path]) -> RawRegistry:
        """Load registry from a list of paths.

        Each path or yaml specifies a dictionary of name -> spec.
        """
        registry: RawRegistry = {}
        for path in paths:
            logging.info(f"Loading registry from {path}")
            if os.path.exists(path):
                if os.path.isdir(path):
                    self._process_directory(registry, path)
                else:
                    self._process_file(registry, path)
        return registry

    @functools.cached_property
    def _completion_fns(self) -> RawRegistry:
        return self._load_registry([p / "completion_fns" for p in self._registry_paths])

    @functools.cached_property
    def _eval_sets(self) -> RawRegistry:
        return self._load_registry([p / "eval_sets" for p in self._registry_paths])

    @functools.cached_property
    def _evals(self) -> RawRegistry:
        return self._load_registry([p / "evals" for p in self._registry_paths])

    @functools.cached_property
    def _modelgraded_specs(self) -> RawRegistry:
        return self._load_registry([p / "modelgraded" for p in self._registry_paths])


registry = Registry()

evals/registry.py
GPT-Fathom/GPT-Fathom
"""
This file defines various helper functions for interacting with the OpenAI API.
"""
import concurrent
import logging
import os

import backoff
import openai

EVALS_THREAD_TIMEOUT = float(os.environ.get("EVALS_THREAD_TIMEOUT", "40"))


@backoff.on_exception(
    wait_gen=backoff.expo,
    exception=(
        openai.error.ServiceUnavailableError,
        openai.error.APIError,
        openai.error.RateLimitError,
        openai.error.APIConnectionError,
        openai.error.Timeout,
    ),
    max_value=60,
    factor=1.5,
)
def openai_completion_create_retrying(*args, **kwargs):
    """
    Helper function for creating a completion.
    `args` and `kwargs` match what is accepted by `openai.Completion.create`.
    """
    result = openai.Completion.create(*args, **kwargs)
    if "error" in result:
        logging.warning(result)
        raise openai.error.APIError(result["error"])
    return result


def request_with_timeout(func, *args, timeout=EVALS_THREAD_TIMEOUT, **kwargs):
    """
    Worker thread for making a single request within allotted time.
    """
    while True:
        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
            future = executor.submit(func, *args, **kwargs)
            try:
                result = future.result(timeout=timeout)
                return result
            except concurrent.futures.TimeoutError as e:
                continue


@backoff.on_exception(
    wait_gen=backoff.expo,
    exception=(
        openai.error.ServiceUnavailableError,
        openai.error.APIError,
        openai.error.RateLimitError,
        openai.error.APIConnectionError,
        openai.error.Timeout,
    ),
    max_value=60,
    factor=1.5,
)
def openai_chat_completion_create_retrying(*args, **kwargs):
    """
    Helper function for creating a chat completion.
    `args` and `kwargs` match what is accepted by `openai.ChatCompletion.create`.
    """
    result = request_with_timeout(openai.ChatCompletion.create, *args, **kwargs)
    if "error" in result:
        logging.warning(result)
        raise openai.error.APIError(result["error"])
    return result

evals/utils/api_utils.py
GPT-Fathom/GPT-Fathom
import concurrent
import logging
import os
import time

import backoff
import openai

EVALS_THREAD_TIMEOUT = float(os.environ.get("EVALS_THREAD_TIMEOUT", "40"))
SLEEP_TIME = float(
    os.environ.get("SLEEP_TIME", "1.5")
)  

@backoff.on_exception(
    wait_gen=backoff.expo,
    exception=(
        openai.error.ServiceUnavailableError,
        openai.error.APIError,
        openai.error.RateLimitError,
        openai.error.APIConnectionError,
        openai.error.Timeout,
    ),
    max_value=60,
    factor=1.5,
)
def azure_completion_create_retrying(*args, **kwargs):
    """
    Helper function for creating a completion.
    `args` and `kwargs` match what is accepted by `openai.Completion.create`.
    """
    openai.api_base = "API-BASE"
    openai.api_key = "API-KEY"

    result = openai.Completion.create(*args, **kwargs)
    if "error" in result:
        logging.warning(result)
        raise openai.error.APIError(result["error"])
    time.sleep(SLEEP_TIME)
    return result


def request_with_timeout(func, *args, timeout=EVALS_THREAD_TIMEOUT, **kwargs):
    """
    Worker thread for making a single request within allotted time.
    """
    while True:
        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
            future = executor.submit(func, *args, **kwargs)
            try:
                result = future.result(timeout=timeout)
                return result
            except concurrent.futures.TimeoutError as e:
                continue


@backoff.on_exception(
    wait_gen=backoff.constant,
    exception=(
        openai.error.ServiceUnavailableError,
        openai.error.APIError,
        openai.error.RateLimitError,
        openai.error.APIConnectionError,
        openai.error.Timeout,
        openai.error.InvalidRequestError,
    ),
    interval=5
    # max_value=60,
)
def azure_chat_completion_create_retrying(GPT_4: bool = False, 
                                          frontend: bool = False,
                                          *args, **kwargs):
    """
    Helper function for creating a chat completion.
    `args` and `kwargs` match what is accepted by `openai.ChatCompletion.create`.
    """
    openai.api_base = "API-BASE"
    openai.api_key = "API-KEY"
    result = request_with_timeout(openai.ChatCompletion.create, *args, **kwargs)
    time.sleep(SLEEP_TIME)
    if "error" in result:
        logging.warning(result)
        raise openai.error.APIError(result["error"])
    return result

evals/utils/azure_utils.py
GPT-Fathom/GPT-Fathom
def last_boxed_only(sample):
    """
    Given a (q,a) sample, filter the answers so that they only contain
    the last \boxed{...} or \fbox{...} element
    """
    q, a = sample
    a = last_boxed_only_string(a)
    if a == None:
        return None
    return (q, a)


def last_boxed_only_string(string):
    idx = string.rfind("\\boxed")
    if idx < 0:
        idx = string.rfind("\\fbox")
        if idx < 0:
            return None

    i = idx
    right_brace_idx = None
    num_left_braces_open = 0
    while i < len(string):
        if string[i] == "{":
            num_left_braces_open += 1
        if string[i] == "}":
            num_left_braces_open -= 1
            if num_left_braces_open == 0:
                right_brace_idx = i
                break
        i += 1

    if right_brace_idx == None:
        retval = None
    else:
        retval = string[idx : right_brace_idx + 1]

    return retval


def only_until_first_boxed_from_tokens(string, tokens):
    idx = string.find("\\boxed")
    if idx < 0:
        idx = string.find("\\fbox")
        if idx < 0:
            return None

    cum_length = 0
    for i, t in enumerate(tokens):
        cum_length += len(t)
        if cum_length >= idx:
            break

    return tokens[:i]


def clean_numbers(sample):
    if not sample:
        return None
    new_sample = list()
    for s in sample:
        new_sample.append(_clean_numbers(s))

    return tuple(new_sample)


def _clean_numbers(string):
    """
    Clean Numbers in the given string

    >>> _clean_numbers(None, "Hello 123")
    'Hello 123'
    >>> _clean_numbers(None, "Hello 1234")
    'Hello 1,234'
    >>> _clean_numbers(None, "Hello 1234324asdasd")
    'Hello 1,234,324asdasd'
    """
    num_prev_digits = 0
    new_string = ""
    for i, c in enumerate(string):
        # isdigit() doesnt work here because of weird unicode chars.
        if c in {"1", "2", "3", "4", "5", "6", "7", "8", "9", "0"}:
            num_prev_digits += 1
        else:
            if num_prev_digits > 3:
                # Some fixing
                string_number = new_string[-num_prev_digits:]
                new_string = new_string[:-num_prev_digits] + "{0:,}".format(
                    int(string_number)
                )
            num_prev_digits = 0
        new_string += c

    if num_prev_digits > 3:
        # Some fixing
        string_number = new_string[-num_prev_digits:]
        new_string = new_string[:-num_prev_digits] + "{0:,}".format(int(string_number))

    return new_string


def remove_boxed(s):
    left = "\\boxed{"
    try:
        assert s[: len(left)] == left
        assert s[-1] == "}"
        return s[len(left) : -1]
    except:
        return None


def _fix_fracs(string):
    substrs = string.split("\\frac")
    new_str = substrs[0]
    if len(substrs) > 1:
        substrs = substrs[1:]
        for substr in substrs:
            new_str += "\\frac"
            if substr[0] == "{":
                new_str += substr
            else:
                try:
                    assert len(substr) >= 2
                except:
                    return string
                a = substr[0]
                b = substr[1]
                if b != "{":
                    if len(substr) > 2:
                        post_substr = substr[2:]
                        new_str += "{" + a + "}{" + b + "}" + post_substr
                    else:
                        new_str += "{" + a + "}{" + b + "}"
                else:
                    if len(substr) > 2:
                        post_substr = substr[2:]
                        new_str += "{" + a + "}" + b + post_substr
                    else:
                        new_str += "{" + a + "}" + b
    string = new_str
    return string


def _fix_a_slash_b(string):
    if len(string.split("/")) != 2:
        return string
    a = string.split("/")[0]
    b = string.split("/")[1]
    try:
        a = int(a)
        b = int(b)
        assert string == "{}/{}".format(a, b)
        new_string = "\\frac{" + str(a) + "}{" + str(b) + "}"
        return new_string
    except:
        return string


def _remove_right_units(string):
    # "\\text{ " only ever occurs (at least in the val set) when describing units
    if "\\text{ " in string:
        splits = string.split("\\text{ ")
        assert len(splits) == 2
        return splits[0]
    else:
        return string


def _fix_sqrt(string):
    if "\\sqrt" not in string:
        return string
    splits = string.split("\\sqrt")
    new_string = splits[0]
    for split in splits[1:]:
        if split[0] != "{":
            a = split[0]
            new_substr = "\\sqrt{" + a + "}" + split[1:]
        else:
            new_substr = "\\sqrt" + split
        new_string += new_substr
    return new_string


def _strip_string(string):
    # linebreaks
    string = string.replace("\n", "")
    # print(string)

    # remove inverse spaces
    string = string.replace("\\!", "")
    # print(string)

    # replace \\ with \
    string = string.replace("\\\\", "\\")
    # print(string)

    # replace tfrac and dfrac with frac
    string = string.replace("tfrac", "frac")
    string = string.replace("dfrac", "frac")
    # print(string)

    # remove \left and \right
    string = string.replace("\\left", "")
    string = string.replace("\\right", "")
    # print(string)

    # Remove circ (degrees)
    string = string.replace("^{\\circ}", "")
    string = string.replace("^\\circ", "")

    # remove dollar signs
    string = string.replace("\\$", "")

    # remove units (on the right)
    string = _remove_right_units(string)

    # remove percentage
    string = string.replace("\\%", "")
    string = string.replace("\%", "")

    # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
    string = string.replace(" .", " 0.")
    string = string.replace("{.", "{0.")
    # if empty, return empty string
    if len(string) == 0:
        return string
    if string[0] == ".":
        string = "0" + string

    # to consider: get rid of e.g. "k = " or "q = " at beginning
    if len(string.split("=")) == 2:
        if len(string.split("=")[0]) <= 2:
            string = string.split("=")[1]

    # fix sqrt3 --> sqrt{3}
    string = _fix_sqrt(string)

    # remove spaces
    string = string.replace(" ", "")

    # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
    string = _fix_fracs(string)

    # manually change 0.5 --> \frac{1}{2}
    if string == "0.5":
        string = "\\frac{1}{2}"

    # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
    string = _fix_a_slash_b(string)

    return string


def is_equiv(str1, str2, verbose=False):
    if str1 is None and str2 is None:
        print("WARNING: Both None")
        return True
    if str1 is None or str2 is None:
        return False

    try:
        ss1 = _strip_string(str1)
        ss2 = _strip_string(str2)
        if verbose:
            print(ss1, ss2)
        return ss1 == ss2
    except:
        return str1 == str2

evals/utils/math_util.py
GPT-Fathom/GPT-Fathom
"""
This file defines miscellanous utilities.
"""
import functools
import importlib
from typing import Any


def t(duration: float) -> str:
    if duration is None:
        return "n/a"
    if duration < 1:
        return f"{(1000*duration):0.3f}ms"
    elif duration < 60:
        return f"{duration:0.3f}s"
    else:
        return f"{duration//60}min{int(duration%60)}s"


def make_object(object_ref: str, *args: Any, **kwargs: Any) -> Any:
    modname, qualname_separator, qualname = object_ref.partition(":")
    obj = importlib.import_module(modname)
    if qualname_separator:
        for attr in qualname.split("."):
            obj = getattr(obj, attr)
    return functools.partial(obj, *args, **kwargs)

evals/utils/misc.py
GPT-Fathom/GPT-Fathom
"""
This file defines utilities for connecting to and interacting with Snowflake.
Familiarity with this file should not be needed even if working with Snowflake.
"""
import logging
import os
import time
from contextlib import contextmanager
from typing import Optional

logger = logging.getLogger(__name__)


def _first_not_none(*args):
    for arg in args:
        if arg is not None:
            return arg
    return None


class SnowflakeError(Exception):
    pass


class SnowflakeConnection:
    def __init__(
        self,
        autocommit=True,
        account: Optional[str] = None,
        user: Optional[str] = None,
        password: Optional[str] = None,
        database: Optional[str] = None,
        warehouse: Optional[str] = None,
        paramstyle="pyformat",
    ):
        self.account = _first_not_none(account, os.environ.get("SNOWFLAKE_ACCOUNT"))
        self.database = _first_not_none(database, os.environ.get("SNOWFLAKE_DATABASE"))

        self.user = _first_not_none(
            user,
            os.environ.get("SNOWFLAKE_USERNAME"),
        )
        self.password = _first_not_none(
            password,
            os.environ.get("SNOWFLAKE_PASSWORD"),
        )

        if self.user is None and self.password is None:
            self.user = os.environ["USER"]
            self.use_browser_auth = True
        else:
            self.use_browser_auth = False

        self.autocommit = autocommit
        self.warehouse = warehouse
        self.paramstyle = paramstyle

        self.ctx = None

    def _ensure_connected(self):
        if self.ctx is not None:
            return
        import snowflake.connector

        logging.getLogger("snowflake").setLevel(logging.WARNING)
        # Snowflake prints a scary "Don't know how to construct
        # ResultBatches from response..." error when doing a select
        # with no results.
        logging.getLogger("snowflake.connector.result_batch").setLevel(logging.CRITICAL)

        try:
            self.ctx = snowflake.connector.connect(
                user=self.user,
                account=self.account,
                database=self.database,
                schema="public",
                password=self.password,
                authenticator="externalbrowser"
                if self.use_browser_auth
                else "snowflake",
                autocommit=self.autocommit,
                client_prefetch_thread=16,
                client_session_keep_alive=True,
                warehouse=self.warehouse,
                paramstyle=self.paramstyle,
            )
        except snowflake.connector.errors.DatabaseError as e:
            raise SnowflakeError(
                f"""Failed to connect to database: {e}
(HINT: if running on a server, you may want to set SNOWFLAKE_PASSWORD=... to use password authentication)"""
            )

    def cursor(self, *args, **kwargs):
        self._ensure_connected()
        cs = self.ctx.cursor(*args, **kwargs)
        return cs

    @contextmanager
    def __call__(self, *args, **kwargs):
        cs = self.cursor(*args, **kwargs)
        try:
            yield cs
        finally:
            cs.close()

    def query(self, *args, many=False, pandas_out=False, list_out=False, **kwargs):
        with self() as cs:
            if many:
                cs.executemany(*args, **kwargs)
            else:
                cs.execute(*args, **kwargs)
            if pandas_out:
                return cs.fetch_pandas_all()
            elif list_out:
                return cs.fetchall()

    def robust_query(self, max_trials: Optional[int] = None, *args, **kwargs):
        from snowflake.connector.errors import OperationalError, ProgrammingError

        ntrials = 0
        while True:
            try:
                return self.query(*args, **kwargs)
            except (OperationalError, ProgrammingError) as e:
                if max_trials is not None and ntrials >= max_trials:
                    raise
                logger.info(f"Snowflake insert failed, will retry in 5s {e}")
                ntrials += 1
                time.sleep(5)

evals/utils/snowflake.py
GPT-Fathom/GPT-Fathom
from typing import Union

from evals.api import CompletionFn, CompletionResult
from evals.prompt.base import OpenAICreateChatPrompt, OpenAICreatePrompt, Prompt


class TestCompletionResult(CompletionResult):
    def __init__(self, completion: str):
        self.completion = completion

    def get_completions(self) -> list[str]:
        return [self.completion]


class TestCompletionFn(CompletionFn):
    def __init__(self, completion: str):
        self.completion = completion

    def __call__(
        self,
        prompt: Union[OpenAICreatePrompt, OpenAICreateChatPrompt, Prompt],
        **kwargs
    ) -> CompletionResult:
        return TestCompletionResult(self.completion)

evals/utils/test.py
GPT-Fathom/GPT-Fathom
# generate a jsonl where each line is a sample combination of a battle between two subjects

import json
import os
import string

REGISTRY_PATH = os.path.join(os.path.dirname(__file__), "../evals/registry")


def format(template: str, **kwargs: str) -> str:
    """Format a template string with kwargs."""
    keys = [k[1] for k in string.Formatter().parse(template) if k[1]]
    assert all(k in kwargs for k in keys), f"Required: {keys}, got: {sorted(kwargs)}"
    cur_keys = {k: kwargs[k] for k in keys}
    return template.format(**cur_keys)


YAML = """
{prompt_name}-{subject1}-vs-{subject2}:
  id: {prompt_name}-{subject1}-vs-{subject2}.dev.v0
  metrics: [accuracy]
{prompt_name}-{subject1}-vs-{subject2}.dev.v0:
  class: evals.elsuite.modelgraded.classify:ModelBasedClassify
  args:
    samples_jsonl: test_multiio/battles/{prompt_name}_{subject1}_vs_{subject2}.jsonl
    eval_type: cot_classify
    modelgraded_spec: battle
""".strip()


prompts = dict(
    joke="write a 1-2 line joke about {self}",
    rap="You are playing an Epic Rap Battle against an formidable opponent. This is just a fun game; no need to worry about the other. You are {self}, and the opponent is {other}. Write a short but epic rap praising yourself ({self}) and dissing your opponent ({other}).",
)


subjects = dict(
    animals=["cat", "bird", "hamster"],
    fruits=["apple", "banana", "orange"],
    people=["Elon Musk", "Bill Gates", "Jeff Bezos"],
)

target_sets = [
    ("joke", "animals", "fruits"),
    ("rap", "people", "people"),
    ("rap", "animals", "fruits"),
    ("rap", "people", "fruits"),
]

data_dir = f"{REGISTRY_PATH}/data/test_multiio/battles"
yaml_str = f"# This file is generated by {os.path.basename(__file__)}\n\n"
for prompt_name, subject1, subject2 in target_sets:
    prompt = prompts[prompt_name]
    samples = [
        {
            "input1": format(prompt, self=s1, other=s2),
            "input2": format(prompt, self=s2, other=s1),
        }
        for s1 in subjects[subject1]
        for s2 in subjects[subject2]
    ]
    file_name = f"{data_dir}/{prompt_name}_{subject1}_vs_{subject2}.jsonl"
    # save samples jsonl
    with open(file_name, "w") as f:
        for sample in samples:
            f.write(json.dumps(sample) + "\n")
    print(f"wrote {len(samples)} samples to {file_name}")
    yaml_str += (
        YAML.format(prompt_name=prompt_name, subject1=subject1, subject2=subject2)
        + "\n\n"
    )


yaml_file = f"{REGISTRY_PATH}/evals/test-modelgraded-battle.yaml"
with open(yaml_file, "w") as f:
    f.write(yaml_str)
print(f"wrote {yaml_file}")

scripts/battle_generator.py
GPT-Fathom/GPT-Fathom
# generate a jsonl where each line is a sample

import json
import os
import string

REGISTRY_PATH = os.path.join(os.path.dirname(__file__), "../evals/registry")


def format(template: str, **kwargs: str) -> str:
    """Format a template string with kwargs."""
    keys = [k[1] for k in string.Formatter().parse(template) if k[1]]
    assert all(k in kwargs for k in keys), f"Required: {keys}, got: {sorted(kwargs)}"
    cur_keys = {k: kwargs[k] for k in keys}
    return template.format(**cur_keys)


YAML = """
mg-{prompt_name}-{subject}:
  id: mg-{prompt_name}-{subject}.dev.v0
  metrics: [accuracy]
mg-{prompt_name}-{subject}.dev.v0:
  class: evals.elsuite.modelgraded.classify:ModelBasedClassify
  args:
    samples_jsonl: test_modelgraded/{prompt_name}_{subject}.jsonl
    eval_type: {eval_type}
    modelgraded_spec: {modelgraded_spec}
""".strip()


unlabeled_prompts = dict(
    humor=dict(
        modelgraded_spec="humor_jp",
        prompt="{subject}で数行の大喜利を書いて。敬語じゃなく関西弁で吉本のM1チャンピョンみたいに書いて。",
        eval_type="cot_classify_jp",
    ),
)


subjects = dict(
    animals=["cat", "bird", "hamster"],
    fruits=["apple", "banana", "orange"],
    people=["Elon Musk", "Bill Gates", "Jeff Bezos"],
    people_jp=[
        "イーロン・マスク",
        "ビル・ゲイツ",
        "ジェフ・ベゾス",
        "アルベルト・アインシュタイン",
        "ニコラ・テスラ",
        "レオナルド・ダ・ヴィンチ",
        "マハトマ・ガンジー",
        "ウィリアム・シェイクスピア",
        "スティーブ・ジョブズ",
        "ビル・ゲイツ",
        "マイケル・ジャクソン",
        "マダム・キュリー",
        "ジョン・F・ケネディ",
        "マーティン・ルーサー・キング・ジュニア",
        "ネルソン・マンデラ",
        "プラトン",
        "アリストテレス",
        "フィリップ・コッタウェイ",
        "ニール・アームストロング",
        "レオ・トルストイ",
        "マルコ・ポーロ",
        "ウィリアム・テル",
        "モーツァルト",
        "ベートーベン",
        "ショパン",
        "ダンテ・アリギエーリ",
        "フランツ・カフカ",
        "ガリレオ・ガリレイ",
        "アイザック・ニュートン",
        "チャールズ・ダーウィン",
        "フリードリヒ・ニーチェ",
        "シェイクスピア",
        "オスカー・ワイルド",
        "アーサー・コナン・ドイル",
        "アガサ・クリスティ",
        "J・K・ローリング",
        "トルーマン・カポーティ",
        "アルフレッド・ヒッチコック",
        "ウォルト・ディズニー",
        "アンディ・ウォーホル",
        "ピカソ",
        "ミケランジェロ",
        "レオナルド・フィボナッチ",
        "アルキメデス",
        "マルコム・X",
        "ジョージ・ワシントン",
        "エイブラハム・リンカーン",
        "フランクリン・D・ルーズベルト",
        "ワシントン・アーヴィング",
        "マーク・トウェイン",
        "フィリップ・K・ディック",
        "ジョージ・オーウェル",
        "トーマス・モア",
        "ハンス・クリスチャン・アンデルセン",
        "グリム兄弟",
        "アレクサンドル・デュマ",
        "ビクトル・ユーゴー",
        "エミール・ゾラ",
        "フランツ・シューベルト",
        "ゲオルク・フリードリヒ・ヘンデル",
        "ヨハン・セバスチャン・バッハ",
        "ルートヴィヒ・ヴァン・ベートーヴェン",
        "ヨハネス・ブラームス",
        "ロベルト・シューマン",
        "ヨハン・シュトラウス2世",
        "イーロン・マスク",
        "スティーブン・ホーキング",
        "リチャード・ファインマン",
        "アラン・チューリング",
        "ニール・デグラス・タイソン",
        "マイケル・ファラデー",
        "スティーブン・スピルバーグ",
        "クリストファー・ノーラン",
        "スタン・リー",
        "ジョージ・ルーカス",
        "ウィリアム・ゴールディング",
        "ジョージ・オーウェル",
        "エルンスト・ヘッケル",
        "ルイ・パスツール",
        "カール・セーガン",
        "アンリ・ベルクソン",
        "ミハイル・バクーニン",
        "ハンス・モルゲンソー",
        "アンドレ・マルロー",
        "シモーヌ・ド・ボーヴォワール",
        "ベルトルト・ブレヒト",
        "ジャン＝ポール・サルトル",
        "フリードリヒ・ヘーゲル",
        "マックス・ウェーバー",
        "マルクス・アウレリウス",
        "レフ・トルストイ",
        "アントン・チェーホフ",
        "フョードル・ドストエフスキー",
        "トルストイ",
        "ウィリアム・フォークナー",
        "エルネスト・ヘミングウェイ",
        "アーサー・ミラー",
        "テネシー・ウィリアムズ",
        "サミュエル・ベケット",
        "ハロルド・ピンター",
        "フランツ・カフカ",
        "ジョージ・バーナード・ショー",
        "ウィリアム・ゴールディング",
        "ジャック・ケルアック",
        "エドガー・アラン・ポー",
        "ハーマン・メルヴィル",
        "ジョセフ・コンラッド",
        "アーサー・コナン・ドイル",
        "ジョン・スタインベック",
        "ジェームズ・ジョイス",
        "バージニア・ウルフ",
        "トマス・マン",
        "フランツ・カフカ",
        "ヘルマン・ヘッセ",
        "ゲオルク・ヴィルヘルム・フリードリヒ・ヘーゲル",
        "エマニュエル・カント",
        "ジャン＝ジャック・ルソー",
        "ジョン・ロック",
        "トマス・ホッブズ",
        "ジョン・デューイ",
        "ジョン・スチュアート・ミル",
        "ニコロ・マキャヴェッリ",
        "モンテスキュー",
        "ルソー",
        "プラトン",
        "アリストテレス",
        "サー・アイザック・ニュートン",
    ],
)
# remove duplicates
subjects = {k: list(set(v)) for k, v in subjects.items()}

unlabeled_target_sets = [
    ("humor", "people_jp"),
]

data_dir = f"{REGISTRY_PATH}/data/test_modelgraded"
yaml_str = f"# This file is generated by {os.path.basename(__file__)}\n\n"
evals = []
for prompt_name, subject in unlabeled_target_sets:
    prompt = unlabeled_prompts[prompt_name]["prompt"]
    samples = [{"input": format(prompt, subject=s)} for s in subjects[subject]]
    file_name = f"{data_dir}/{prompt_name}_{subject}.jsonl"
    # save samples jsonl
    with open(file_name, "wb") as f:
        for sample in samples:
            # f.write(json.dumps(sample) + "\n")
            json_data = json.dumps(sample, ensure_ascii=False)
            f.write(json_data.encode("utf-8"))
            f.write(b"\n")
    print(f"wrote {len(samples)} samples to {file_name}")
    yaml_str += (
        YAML.format(
            prompt_name=prompt_name,
            subject=subject,
            modelgraded_spec=unlabeled_prompts[prompt_name]["modelgraded_spec"],
            eval_type=unlabeled_prompts[prompt_name]["eval_type"],
        )
        + "\n\n"
    )
    evals += [f"mg-{prompt_name}-{subject}: {file_name}"]


yaml_file = f"{REGISTRY_PATH}/evals/test-modelgraded-generated.yaml"
with open(yaml_file, "w") as yf:
    yf.write(yaml_str)
print(f"wrote {yaml_file}")
for e in evals:
    print(e)

scripts/modelgraded_generator.py
GPT-Fathom/GPT-Fathom
"""
    Usage: python scripts/task_identification_generator.py
"""

import json
import random
from typing import Literal

random.seed(42)

SYMBOLS = list("abcdefghijklmnopqrstuvwxyz")
DELIMETER = "->"
INSTRUCTION = 'Figure out the pattern in the below examples, and then answer with just "foo" or "bar".'
TASK_NAME = "pattern_identification"


def generate_example() -> tuple[str, list[str], Literal["foo", "bar"]]:
    num_symbols = int(len(SYMBOLS) / 2)
    target_symbol = random.choice(SYMBOLS)
    symbol_list = random.sample(SYMBOLS, num_symbols)
    target: Literal["foo", "bar"] = "foo" if target_symbol in symbol_list else "bar"
    return (target_symbol, symbol_list, target)


def generate_exemplars_str(num_exemplars: int = 8) -> str:
    exemplars = [generate_example() for _ in range(num_exemplars)]
    exemplars_str = [
        f"({exemplar[0]}, {exemplar[1]}) {DELIMETER} {exemplar[2]}".replace("'", "")
        for exemplar in exemplars
    ]
    return "\n".join([INSTRUCTION] + exemplars_str)


def generate_eval_examples(
    num_eval_examples: int = 250,
) -> tuple[list[str], list[Literal["foo", "bar"]]]:
    eval_examples = [generate_example() for _ in range(num_eval_examples)]
    eval_examples_str = [
        f"{generate_exemplars_str()}\n({example[0]}, {example[1]}) {DELIMETER}".replace(
            "'", ""
        )
        for example in eval_examples
    ]
    targets: list[Literal["foo", "bar"]] = [example[2] for example in eval_examples]
    return eval_examples_str, targets


if __name__ == "__main__":
    eval_examples_str, targets = generate_eval_examples()
    output_path = f"evals/registry/data/{TASK_NAME}/samples.v0.jsonl"
    with open(output_path, "w") as writer:
        for eval_example_str, target in zip(eval_examples_str, targets):
            d = {
                "input": [
                    {"role": "system", "content": "You are a helpful assistant."},
                    {"role": "user", "content": eval_example_str},
                ],
                "ideal": target,
            }
            writer.write(json.dumps(d) + "\n")
    print(f"{len(eval_examples_str)} lines written to {output_path}.")

scripts/pattern_identification_generator.py
GPT-Fathom/GPT-Fathom
from typing import List
from unittest.mock import MagicMock

import numpy as np
import pytest

from evals import metrics


@pytest.mark.parametrize(
    "event_labels, expected",
    [
        ([True, True], 1.0),
        ([True, False, False], 0.333),
        ([False, False], 0.0),
        ([], np.nan),
    ],
)
def test_get_accuracy(
    event_labels: List[bool],
    expected: float,
) -> None:
    events = [MagicMock(data={"correct": value}) for value in event_labels]
    np.testing.assert_allclose(expected, metrics.get_accuracy(events), rtol=1e-3)

tests/unit/evals/test_metrics.py
GPT-Fathom/GPT-Fathom