SciPhi is a simple framework for generating synthetic / fine-tuning data, and for robust evaluation of LLMs.
"""A module to check that all stock configurations perform as expected."""
import os

from sciphi.config import DataConfig
from sciphi.core.utils import get_data_config_dir
from sciphi.prompt import PromptGenerator

NUM_SAMPLES = 1_000_000
if __name__ == "__main__":
    data_config = DataConfig(
        os.path.join(
            get_data_config_dir(),
            "textbooks_are_all_you_need_basic_split/main.yaml",
        )
    )
    prompt_generator = PromptGenerator(
        data_config.config,
        data_config.prompt_templates,
        data_config.prompt_template_input_dependencies,
        data_config.prompt_dataset_dependencies,
        data_config.prompt_inputs,
    )
    for i in range(NUM_SAMPLES):
        prompt_generator.generate_prompt()

.github/scripts/check_prompt_generators.py
emrgnt-cmplxty/sciphi
from sciphi.config.base import DataGeneratorMode
from sciphi.config.config import DataConfig

__all__ = ["DataGeneratorMode", "DataConfig"]

sciphi/config/__init__.py
emrgnt-cmplxty/sciphi
"""A module for base classes and enums for makers."""
from enum import Enum


class DataGeneratorMode(Enum):
    """What mode is the data maker in?"""

    SYNTHETIC = "synthetic"
    FROM_HF_DATASET = "from_hf_dataset"

sciphi/config/base.py
emrgnt-cmplxty/sciphi
import os
from typing import Optional

import yaml

from sciphi.config.base import DataGeneratorMode


class DataConfig:
    """Reads the `main` YAML config file and constructs a data config."""

    def __init__(
        self,
        main_config_path: str,
        generator_mode_field: str = "generator_mode",
        prompt_templates_field: str = "prompt_templates",
        prompt_inputs_field: str = "prompt_template_inputs",
        prompt_template_input_dependencies_field: str = "prompt_template_input_dependencies",
        outer_prompt_format_field: str = "outer_prompt_format",
        downstream_configs_field: str = "config_files",
        dataset_name_field: str = "dataset_name",
        prompt_dataset_dependencies_field: str = "prompt_dataset_dependencies",
    ):
        with open(main_config_path, "r") as file:
            self.config = yaml.safe_load(file)
        self.main_path = os.path.dirname(main_config_path)
        self.generator_mode: str = self.config.pop(generator_mode_field)
        # Unpack the prompt config
        self.prompt_templates: dict[str, int] = self.config.pop(
            prompt_templates_field
        )
        self.prompt_inputs: list[str] = self.config.pop(prompt_inputs_field)
        self.prompt_template_input_dependencies: dict[
            str, str
        ] = self.config.pop(prompt_template_input_dependencies_field)
        self.outer_prompt_format: str = self.config.pop(
            outer_prompt_format_field
        )
        self.downstream_configs: dict[str, str] = self.config.pop(
            downstream_configs_field
        )
        self.prompt_dataset_dependencies: dict[str, str] = self.config.pop(
            prompt_dataset_dependencies_field, {}
        )
        self.dataset_name: Optional[str] = self.config.pop(
            dataset_name_field, None
        )

        if (
            self.generator_mode == DataGeneratorMode.FROM_HF_DATASET
            and not self.prompt_dataset_dependencies
        ):
            raise ValueError(
                f"Must specify a dataset name when using {self.generator_mode}."
            )

        self._load_configs()

        # TODO - Add smart config validation
        # self._validate_config()

    def _load_configs(self) -> None:
        """Load the sub-configs into the main config."""

        if not self.downstream_configs:
            return

        for sub_config_name, weight in self.downstream_configs.items():
            with open(
                os.path.join(self.main_path, f"{sub_config_name}.yaml"), "r"
            ) as sub_config_file:
                sub_config = yaml.safe_load(sub_config_file)
                # Merge the sub_config into the main config
                for key in sub_config.keys():
                    entry = sub_config[key]

                    # Unroll dependencies (only single layers are supported now)
                    if (
                        self.prompt_template_input_dependencies
                        and key in self.prompt_template_input_dependencies
                    ):
                        entry = entry[
                            self.prompt_template_input_dependencies[key]
                        ]

                    if key in self.config and isinstance(entry, dict):
                        # Verify that the sub_config does not overlap with the main config
                        intersection = set(entry.keys()).intersection(
                            set(self.config[key].keys())
                        )
                        if intersection:
                            raise ValueError(
                                f"Sub-config {key} overlaps with main config, resulting in an intersection {intersection}. Note, every key must be unique in the unrolled dictionary."
                            )
                        # Weight the sub_config entries by the weight
                        # If value is a dictionary, then recursively weight the values
                        weighted_entry = {
                            k: v * weight
                            if isinstance(v, int)
                            else {ks: vs * weight for ks, vs in v.items()}
                            for k, v in entry.items()
                        }
                        # Merge the weighted sub_config into the main config
                        self.config[key] = {
                            **weighted_entry,
                            **self.config[key],
                        }
                    elif (
                        key in self.config and isinstance(entry, str)
                    ) or key not in self.config:
                        self.config[key] = entry
                    else:
                        # Only dicts and strings are supported, raise an error otherwise
                        raise ValueError(
                            f"Unexpected type for config entry {key}: {type(entry)}"
                        )

sciphi/config/config.py
emrgnt-cmplxty/sciphi
from sciphi.core.base import ProviderName

__all__ = [
    "ProviderName",
]

sciphi/core/__init__.py
emrgnt-cmplxty/sciphi
"""Base classes and enums for zero-shot replication problems."""
from enum import Enum


class ProviderName(Enum):
    """Specifies the name of the LLM provider"""

    OPENAI = "openai"
    ANTHROPIC = "anthropic"
    HUGGING_FACE = "hugging-face"
    LLAMA_INDEX = "llama-index"
    VLLM = "vllm"

sciphi/core/base.py
emrgnt-cmplxty/sciphi
"""A module which defines utilities for the library."""
import json
import logging
import os

import pandas as pd


def load_file_or_raise(path: str) -> pd.DataFrame:
    """Utility function to load a file or raise an error if not found."""

    try:
        file_extension = os.path.splitext(path)[-1].lower()
        if file_extension == ".csv":
            return pd.read_csv(path)
        elif file_extension == ".jsonl":
            with open(path, "r", encoding="utf-8") as file:
                return pd.DataFrame(
                    json.loads(line) for line in file if line.strip()
                )
        else:
            raise ValueError(f"Unsupported file format: {file_extension}")
    except FileNotFoundError as e:
        raise FileNotFoundError(
            f"Please check the expected data at {path}."
        ) from e


def load_existing_jsonl(file_path: str) -> list[dict]:
    """Load existing results from a jsonl file."""
    if os.path.exists(file_path):
        with open(file_path, "r") as json_file:
            return [json.loads(line) for line in json_file]
    return []


def get_configured_logger(name: str, log_level: str) -> logging.Logger:
    """Get a configured logger."""
    log_level = getattr(logging, log_level.upper(), "INFO")
    logging.basicConfig(
        level=log_level, format="%(asctime)s - %(levelname)s - %(message)s"
    )
    return logging.getLogger(name)


def get_root_dir() -> str:
    """Get the path to the root of the code repository."""
    script_dir = os.path.dirname(os.path.realpath(__file__))
    return os.path.join(script_dir, "..", "..")


def get_data_dir() -> str:
    """Get the path to the root of the config directory."""
    script_dir = os.path.dirname(os.path.realpath(__file__))
    return os.path.join(script_dir, "..", "data")


def get_data_config_dir() -> str:
    """Get the path to the root of the config directory."""
    script_dir = os.path.dirname(os.path.realpath(__file__))
    return os.path.join(script_dir, "..", "data", "stock_config")


def get_data_raw_dir() -> str:
    """Get the path to the root of the raw data directory."""
    script_dir = os.path.dirname(os.path.realpath(__file__))
    return os.path.join(script_dir, "..", "data", "stock_raw")

sciphi/core/utils.py
emrgnt-cmplxty/sciphi
"""Run the dataset generation."""
import argparse
import hashlib
import os
import random
import secrets
import time

from sciphi.config import DataConfig, DataGeneratorMode
from sciphi.core.utils import (
    get_configured_logger,
    get_data_config_dir,
    get_root_dir,
)
from sciphi.examples.helpers import (
    gen_llm_config,
    parse_arguments,
    prep_for_file_path,
)
from sciphi.interface import InterfaceManager, ProviderName
from sciphi.llm import LLMConfigManager
from sciphi.makers import DataMaker
from sciphi.prompt import (
    Prompt,
    PromptGenerator,
    PromptManager,
    PromptStructure,
)
from sciphi.writers import JsonlDataWriter

OUTPUT_FILE_NAME = "{RUN_NAME}__provider_eq_{PROVIDER}__model_eq_{MODEL}__version_eq_{VERSION}{EXTRA}.jsonl"


def get_output_path(args: argparse.Namespace) -> str:
    """Get the output path for the given arguments."""

    def generate_random_hash() -> str:
        """Generate a random hash."""
        # Generate a random token in byte format and then convert to SHA256 hash
        random_token = secrets.token_bytes(32)  # 32 bytes = 256 bits
        return hashlib.sha256(random_token).hexdigest()

    output_dir = os.path.join(
        get_root_dir(),
        prep_for_file_path(args.output_dir),
        prep_for_file_path(args.provider_name),
        prep_for_file_path(args.model_name),
    )
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    # TODO - Fail check if path does not exist after attempted creation

    if not args.output_file_name:
        output_file_name = OUTPUT_FILE_NAME.format(
            **{
                k: prep_for_file_path(v)
                for k, v in {
                    "RUN_NAME": args.run_name,
                    "PROVIDER": str(args.provider_name),
                    "MODEL": args.model_name,
                    "VERSION": args.version,
                    "EXTRA": args.extra_output_file_text
                    or f"_{generate_random_hash()}",
                }.items()
            }
        )
    else:
        output_file_name = args.output_file_name

    return os.path.join(output_dir, output_file_name)


if __name__ == "__main__":
    """Run the synthesis."""
    # Setup local environment
    args = parse_arguments()
    logger = get_configured_logger("sciphi", log_level=args.log_level)

    if args.num_samples % args.batch_size != 0:
        raise ValueError(
            f"Number of samples ({args.num_samples}) must be divisible by batch size ({args.batch_size})."
        )
    if args.num_samples < args.batch_size:
        raise ValueError(
            "Number of samples must be greater than batch size0. Please set --num_samples."
        )

    model_name = args.model_name
    provider_name = ProviderName(args.provider_name)

    logger.info(
        f"Loading ModelName={model_name} from ProviderName={provider_name.value}."
    )

    # Build an LLM and provider interface
    llm_config = LLMConfigManager.get_config_for_provider(
        provider_name
    ).create(**gen_llm_config(args))
    llm_provider = InterfaceManager.get_provider(
        provider_name,
        model_name,
        llm_config,
    )

    # Initialize the prompt generator
    data_config = DataConfig(
        os.path.join(
            get_data_config_dir(), f"{args.example_config}", "main.yaml"
        )
    )
    prompt_generator = PromptGenerator(
        data_config.config,
        data_config.prompt_templates,
        data_config.prompt_template_input_dependencies,
        data_config.prompt_dataset_dependencies,
        data_config.prompt_inputs,
    )
    prompt = PromptManager().get_prompt(data_config.outer_prompt_format)
    if args.prompt_override != "":
        logger.debug(f"Overriding default prompt with: {args.prompt_override}")
        prompt_inputs = args.prompt_override.split(",")
        prompt = Prompt(
            raw_text=prompt_inputs[0],
            expected_inputs=set(prompt_inputs[1:]),
            structure=PromptStructure.SINGLE,
        )

    # Initialize the data maker
    data_maker = DataMaker(
        DataGeneratorMode(data_config.generator_mode),
        prompt_generator,
        prompt,
        # Optional field,
        # currently only used when generator_mode == "from_hf_dataset"
        dataset_name=data_config.dataset_name,
    )

    # Generate & write out the results
    output_path = get_output_path(args)
    logger.debug(f"Writing results to: {output_path}.")
    writer = JsonlDataWriter(output_path)

    seed = int(time.time() * 1_000_000) + os.getpid()
    logger.info(f"Using seed: {seed}")
    random.seed(seed)

    for batch in data_maker.generator(args.batch_size, args.num_samples):
        completions = llm_provider.get_batch_completion(batch)
        for formatted_prompt, completion in zip(batch, completions):
            logger.debug("-" * 100)
            logger.debug(f"Formatted Prompt:\n{formatted_prompt}")
            logger.debug(f"\nCompletion:\n{completion}")
            logger.debug("-" * 100)
            # Write the results using DataWriter
            writer.write(
                [
                    {
                        "formatted_prompt": formatted_prompt,
                        "completion": completion,
                    }
                ]
            )

sciphi/examples/basic_data_gen/runner.py
emrgnt-cmplxty/sciphi
"""Helper functions to be used across example scripts"""
import argparse
import json
import logging
import os
import time
from typing import Generator, Tuple

import requests
import yaml
from requests.auth import HTTPBasicAuth

from sciphi.interface import InterfaceManager, ProviderName
from sciphi.interface.base import LLMInterface
from sciphi.llm import LLMConfigManager


def gen_llm_config(args: argparse.Namespace) -> dict:
    """Constructs the LLM config based on provided arguments."""

    config_args = {
        "provider_name": ProviderName(args.provider_name),
        "model_name": args.model_name,
        "temperature": args.temperature,
        "top_p": args.top_p,
        "top_k": args.top_k,
        "max_tokens_to_sample": args.max_tokens_to_sample,
        "do_sample": args.do_sample,
        "num_beams": args.num_beams,
        "do_stream": args.do_stream,
        "device": args.device,
        "version": args.version,
        "llama_data_dir": args.llama_data_dir,
        "llama_index_name": args.llama_index_name,
        "llama_out_store": args.llama_out_store,
        "llama_load_from_avail_store": args.llama_load_from_avail_store,
        "llama_chunk_size": args.llama_chunk_size,
        "llama_top_k_similarity": args.llama_top_k_similarity,
        "add_model_kwargs": json.loads(args.add_model_kwargs)
        if args.add_model_kwargs
        else None,
        "add_generation_kwargs": json.loads(args.add_generation_kwargs)
        if args.add_generation_kwargs
        else None,
        "add_tokenizer_kwargs": json.loads(args.add_tokenizer_kwargs)
        if args.add_tokenizer_kwargs
        else None,
        "functions": json.loads(args.functions) if args.functions else None,
    }

    # Filter out None values
    return {k: v for k, v in config_args.items() if v is not None}


def prep_for_file_path(in_path: str) -> str:
    """Prepare a string to be used in a file path."""
    return in_path.replace("-", "_").replace(".", "p").replace("/", "_")


def parse_arguments() -> argparse.Namespace:
    """Parse command line arguments."""

    parser = argparse.ArgumentParser(
        description="Parse SciPhi running commands"
    )
    # Run arguments
    parser.add_argument(
        "--run_name",
        type=str,
        default="run_0",
        help="The name of the run.",
    )
    parser.add_argument(
        "--provider_name",
        type=str,
        default="openai",
        help="Which provider to use for zero-shot completions?",
    )
    # Base LLM arguments
    parser.add_argument(
        "--model_name",
        type=str,
        default="gpt-3.5-turbo",
        help="Model name to load from the provider.",
    )
    parser.add_argument(
        "--temperature",
        type=float,
        default=0.7,
        help="Temperature parameter for provided model.",
    )
    parser.add_argument(
        "--top_p",
        type=float,
        default=1,
        help="Top-p parameter for provided model.",
    )
    # Model dependent arguments (each defaults to None)
    parser.add_argument(
        "--top_k",
        type=float,
        default=None,
        help="Top-k parameter for provided model.",
    )
    parser.add_argument(
        "--max_tokens_to_sample",
        type=int,
        default=1_024,
        help="Max tokens to sample for each completion from the provided model.",
    )
    parser.add_argument(
        "--do_sample",
        type=bool,
        default=None,
        help="Should the model run with sampling?",
    )
    parser.add_argument(
        "--num_beams",
        type=int,
        default=None,
        help="The number of beams to use for beam search.",
    )
    parser.add_argument(
        "--do_stream",
        type=bool,
        default=None,
        help="Should the model run with streaming?",
    )
    parser.add_argument(
        "--device",
        type=str,
        default=None,
        help="Which device should the model run on (relevant for local models, like HuggingFace)?",
    )
    parser.add_argument(
        "--add_model_kwargs",
        type=str,
        default=None,
        help="Additional model kwargs for local model generation, in JSON format.",
    )
    parser.add_argument(
        "--add_tokenizer_kwargs",
        type=str,
        default=None,
        help="Additional tokenizer kwargs for local model generation, in JSON format.",
    )
    parser.add_argument(
        "--add_generation_kwargs",
        type=str,
        default=None,
        help="Additional generation kwargs for local model generation, in JSON format.",
    )
    parser.add_argument(
        "--functions",
        type=str,
        default=None,
        help="List of functions which conform to for OpenAI function calling format, in JSON format.",
    )
    parser.add_argument(
        "--version",
        type=str,
        default="0.1.0",
        help="Version of this run.",
    )
    # Llama data arguments
    parser.add_argument(
        "--llama_data_dir",
        type=str,
        default=None,
        help="Which directory to load the LlamaIndex data from?",
    )
    parser.add_argument(
        "--llama_index_name",
        type=str,
        default=None,
        help="Which directory to load the LlamaIndex data from?",
    )
    parser.add_argument(
        "--llama_out_store",
        type=str,
        default=None,
        help="Which directory save the LlamaIndex store to?",
    )
    parser.add_argument(
        "--llama_load_from_avail_store",
        type=bool,
        default=True,
        help="Should the LlamaIndex load from the available store?",
    )
    parser.add_argument(
        "--llama_chunk_size",
        type=int,
        default=None,
        help="What size should the LlamaIndex chunks be?",
    )
    parser.add_argument(
        "--llama_top_k_similarity",
        type=int,
        default=None,
        help="What top k should the LlamaIndex use for similarity?",
    )
    # Prompt arguments
    parser.add_argument(
        "--prompt_override",
        type=str,
        default="",
        help="Override the prompt type for the prompt with a comma separated list. The first entry is the prompt, followed by expected prompt inputs. E.g. `my_prompt requires {input},input`",
    )
    # Data generation arguments
    parser.add_argument(
        "--batch_size",
        type=int,
        default=64,
        help="Number of examples to generate per batch.",
    )
    parser.add_argument(
        "--config_path",
        type=str,
        default=None,
        help="Optional path to the configuration path, if specified the example config is overridden.",
    )
    parser.add_argument(
        "--example_config",
        type=str,
        default="textbooks_are_all_you_need_basic_split",
        help="Which configuration to use for data generation?",
    )
    parser.add_argument(
        "--extra_output_file_text",
        default="",
        help="Extra text to append to the end of the string.",
    )
    parser.add_argument(
        "--log_level",
        type=str,
        default="INFO",
        help="Logging verbosity level.",
    )
    parser.add_argument(
        "--num_samples",
        type=int,
        default=1_024,
        help="Number of samples to generate.",
    )
    parser.add_argument(
        "--output_file_name",
        default=None,
        help="Filename to override the default output file name with.",
    )
    parser.add_argument(
        "--output_dir",
        default="outputs",
        help="Directory to write generated output to.",
    )
    return parser.parse_args()


def setup_logging(log_level: str = "INFO") -> logging.Logger:
    """Set up logging with the given log level."""
    logging.basicConfig(level=log_level.upper())
    return logging.getLogger(__name__)


def get_default_settings_provider(
    provider: str, model_name: str, max_tokens_to_sample=None
) -> LLMInterface:
    """Get the default LLM config and provider for the given provider and model name."""

    provider_name = ProviderName(provider)
    llm_config = LLMConfigManager.get_config_for_provider(
        provider_name
    ).create(max_tokens_to_sample=max_tokens_to_sample, model_name=model_name)

    return InterfaceManager.get_provider(provider_name, model_name, llm_config)


def prep_yaml_line(line: str) -> str:
    """Replace special characters in the YAML string."""
    replacements = {
        "\u201C": '"',
        "\u201D": '"',
        "Ã©": "é",
        "Å": "œ",
        "â": "",
        "Î©": "I",
        "Ï": "I",
        "\x83": "",
        "\x88": "",
        "\x89": "",
        "\x90": "",
        "\x91": "",
        "\x92": "",
        "\x9b": "",
        "\x8f": "",
    }
    for old, new in replacements.items():
        line = line.replace(old, new)

    line = line.replace("\\", "\\\\")
    line = line.replace('"', "'")

    return line


def format_yaml_line(line: str, index: int, split_lines: list[str]) -> str:
    """Format a specific line in the YAML content."""
    line_cut = line.find("- ")
    if line_cut != -1:
        end = len(line) if line[-1] != ":" else len(line) - 1
        line = (
            line[: line_cut + 2]
            + '"'
            + line[line_cut + 2 : end]
            + '"'
            + ("" if end == len(line) else ":")
        )
        if line[-1] != ":" and index != len(split_lines) - 1:
            if "subtopics:" in split_lines[index + 1]:
                line += ":"
    elif index == 2:
        first_non_blank_char = next(
            (i for i, char in enumerate(line) if char != " "), 0
        )
        if first_non_blank_char != 0:
            line = (
                line[:first_non_blank_char]
                + '"'
                + line[first_non_blank_char:]
                + '":'
            )
    return line


def load_yaml_file(yml_file_path: str, do_prep=False) -> dict:
    """Load YAML content from the given file path."""
    try:
        with open(yml_file_path, "r") as yfile:
            if do_prep:
                # Parse and clean YAML content
                yfile_content = yfile.read()
                parsed_content = ""
                split_lines = yfile_content.splitlines()
                for it, line in enumerate(split_lines):
                    if "```yaml" in line:
                        line = line.replace("```yaml", "")
                    elif "```" in line:
                        line = line.replace("```", "")
                    line = prep_yaml_line(line)
                    line = format_yaml_line(line, it, split_lines)
                    parsed_content += line + "\n"
                return yaml.safe_load(parsed_content)
            else:
                return yaml.safe_load(yfile)
    except FileNotFoundError:
        logging.error(f"File not found: {yml_file_path}")
        raise
    except yaml.YAMLError as e:
        logging.error(f"Error {e} while parsing YAML file: {yml_file_path}")
        raise


def save_yaml(content: str, filename: str) -> None:
    """Save content to a YAML file."""
    with open(filename, "w", encoding="utf-8") as file:
        yaml.dump(content, file, allow_unicode=True)


def ensure_directory_exists(directory_path: str) -> None:
    """Ensure that a directory exists, and create it if it doesn't."""
    if not os.path.exists(directory_path):
        os.makedirs(directory_path)


def prase_yaml_completion(yml_content: dict) -> str:
    """Parse and clean YAML content."""
    completion = yml_content.get("completion", "")
    if "```yaml" not in completion:
        raise ValueError("YML not found in completion")

    yml_str = completion.split("```yaml")[1].split("```")[0]

    def clean_yaml_string(yml_str: str) -> str:
        """Clean and format the YAML string."""
        parsed_yml_str = ""
        split_lines = yml_str.splitlines()

        for it, line in enumerate(split_lines):
            line = prep_yaml_line(line)
            line = format_yaml_line(line, it, split_lines)
            parsed_yml_str += line + "\n"

        return parsed_yml_str

    return clean_yaml_string(yml_str)


def wiki_search_api(
    url: str, username: str, password: str, query: str, top_k=10
) -> dict:
    """
    Queries the search API with the provided credentials and query.
    The expected output is a JSON response containing the top_k examples.
    """
    # Make the GET request with basic authentication and the query parameter
    response = requests.get(
        url,
        auth=HTTPBasicAuth(username, password),
        params={"query": query, "k": top_k},
    )

    # Check if the request was successful
    if response.status_code == 200:
        return response.json()["match"]  # Return the JSON response
    else:
        response.raise_for_status()  # Raise an HTTPError if the HTTP request returned an unsuccessful status code
        raise ValueError("Unexpected response from API")


def traverse_config(
    config: dict,
) -> Generator[Tuple[str, str, str, str], None, None]:
    """Traverse the config and yield textbook, chapter, section, subsection names"""

    def get_key(config_dict: dict) -> str:
        """Get the key from a dictionary with a single key-value pair"""
        keys = list(config_dict.keys())
        if not keys:
            raise KeyError("Dictionary is empty, no key found")
        return keys[0]

    textbook_name = get_key(config["textbook"])
    chapters = config["textbook"][textbook_name]["chapters"]

    for chapter in chapters:
        chapter_name = get_key(chapter)
        sections = chapter[chapter_name]["sections"]
        for section in sections:
            section_name = get_key(section)
            subsections = section[section_name]["subsections"]
            if not subsections:
                yield textbook_name, chapter_name, section_name, ""
            for subsection in subsections:
                if isinstance(subsection, str):
                    yield textbook_name, chapter_name, section_name, subsection
                elif isinstance(
                    subsection, dict
                ):  # Additional check if subsections have nested structure
                    subsection_name = get_key(subsection)
                    # Add logic to handle deeper nested structures if needed
                    yield textbook_name, chapter_name, section_name, subsection_name


def with_retry(func, max_retries=3):
    """Attempt to execute the provided function up to max_retries times."""
    for _ in range(max_retries):
        try:
            return func()
        except Exception as e:
            logging.warning(f"Exception encountered: {e}. Retrying...")
            time.sleep(5)
    raise ValueError(
        f"Failed to execute {func.__name__} after {max_retries} retries."
    )

sciphi/examples/helpers.py
emrgnt-cmplxty/sciphi
# type: ignore
"""
MIT OCW Course Data Scraper to syllyabi 'Draft' YAML Generator

Description:
    This script is designed to scrape course data from the MIT OpenCourseWare (OCW) website and 
    generate input YAML files with the help of an LLM.

Usage:
    Command-line interface:
        $ python sciphi/examples/library_of_phi/gen_step_1_draft_syllabi.py run \
            --output_rel_dir=my_output_directory \
            --input_jsonl_filename=my_input_file.jsonl \
            --log_level=DEBUG
Parameters:
    
    provider (str):
        The provider to use for LLM completions.
        Default is 'openai'.
    
    model_name (str):
        The model to use for LLM completions.
        Default is 'gpt-4-0613'.

    data_directory (Optional[str]): 
        The directory the input and output data is to be stored.
        If none, defaults to the directory of this script plus '/raw_data'.

    output_rel_dir (str): 
        The relative directory within the data directory where the generated YAML files will be saved. 
        Default value is 'output_step_1'.

    input_jsonl_filename (str): 
        The name of the input jsonl file containing course data scraped from MIT OCW.
        Default value is 'output_step_1'.

    log_level (str): 
        Logging level for the scraper. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL. 
        Default is 'INFO'.
"""

import json
import logging
import os
from glob import glob
from typing import Optional, Set
import re

import fire
import yaml

from sciphi.examples.helpers import get_default_settings_provider
from sciphi.examples.khan_academy.prompts import SYLLABI_CREATION_PROMPT

from sciphi.examples.helpers import (
    prase_yaml_completion,
    save_yaml,
)


def extract_data_from_record(record: dict[str, str]) -> tuple[dict, str]:
    """Extract and organize data from a given record."""
    context = f"### Course Name:\n{record['course_title'].replace(':', '')}\n"
    topics = {}
    page_contents = record["page_contents"]["syllabus"]

    for topic in page_contents["topics"]:
        context += f"## Page:\n{topic}\n"
        topics[topic] = "\n".join(page_contents["topics"][topic])

        table = "\n".join(page_contents["topics"][topic])
        context += f"Information:\n{table}\n"

    return topics, context


def get_mapped_topics(topics: str) -> dict[str, str]:
    """Get mapped topics based on depth."""
    mapped_topics = {}
    for topic in topics:
        if topics[topic] == 0:
            mapped_topics["category"] = topic
        elif topics[topic] == 1:
            mapped_topics["field"] = topic
        elif topics[topic] == 2:
            mapped_topics["subfield"] = topic
            break
    return mapped_topics


def get_observed_files(output_dir: str) -> Set[str]:
    """Get set of previously observed files."""
    observed_files = set([])
    for yaml_file in glob(f"{output_dir}/**/*.yaml"):
        with open(yaml_file, "r") as existing_file:
            yaml_load = yaml.safe_load(existing_file)
            if "course" in yaml_load:
                for course_name in yaml_load["course"]:
                    _, discipline, field = yaml_file.split("/")
                    field = field.replace(".yaml", "")
                    subfield = yaml_load["course"][course_name]["subfield"]
                    observed_files.add(
                        f"field_{field}_subfield_{subfield}_course_name_{course_name.replace(' ','_')}"
                    )
    for yaml_dumps in glob(f"{output_dir}/*.yaml"):
        observed_files.add(yaml_dumps.split("/")[-1].replace(".yaml", ""))
    return observed_files


def quoted_presenter(dumper, data):
    """Define a custom representer for string scalars that applies quotes."""
    return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"')


class DraftSyllabiYAMLRunner:
    """Runs the generation process for draft syllabi YAMLs."""

    def __init__(
        self,
        provider: str = "openai",
        model_name: str = "gpt-4-0613",
        data_directory: Optional[str] = None,
        output_rel_dir: str = "output_step_1",
        input_jsonl_filename: str = "ka_scraped.jsonl",
        prompt: str = SYLLABI_CREATION_PROMPT,
        log_level: str = "INFO",
    ):
        self.provider = provider
        self.model_name = model_name
        self.data_directory = data_directory
        self.output_rel_dir = output_rel_dir
        self.input_jsonl_filename = input_jsonl_filename
        self.prompt = prompt
        logging.basicConfig(level=getattr(logging, log_level.upper()))

    def run(self) -> None:
        """Run the draft YAML generation process."""
        yaml.add_representer(str, quoted_presenter)

        llm_provider = get_default_settings_provider(
            provider=self.provider, model_name=self.model_name
        )
        if not self.data_directory:
            file_path = os.path.dirname(os.path.abspath(__file__))
            self.data_directory = os.path.join(file_path, "raw_data")

        output_dir = os.path.join(self.data_directory, self.output_rel_dir)
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
        logging.info(f"Saving data to output directory = {output_dir}")

        # The input file path = data directory + input file name
        input_file_path = os.path.join(
            self.data_directory, self.input_jsonl_filename
        )

        with open(input_file_path, "r") as file:
            for line in file:
                try:
                    record = json.loads(line)

                    # Don't use the topics in KA, and all the context is in the string
                    _, context = extract_data_from_record(record)

                    course_name = record["course_title"]

                    # Replace all non-alphanumeric characters with underscores
                    dump_name = re.sub(
                        r"\W+", "", course_name.replace(" ", "_")
                    )

                    if os.path.exists(
                        os.path.join(output_dir, f"{dump_name}.yaml")
                    ):
                        logging.warn(
                            f"Skipping {dump_name} because it was already created..."
                        )
                        continue

                    formatted_prompt = self.prompt.format(
                        course_name=course_name,
                        context=context,
                    )

                    completion = llm_provider.get_completion(formatted_prompt)

                    data_to_save = {
                        "completion": completion,
                        "course_name": course_name,
                    }

                    yaml_completion = prase_yaml_completion(data_to_save)
                    yml_load = yaml.safe_load(yaml_completion)
                    save_yaml(
                        yml_load,
                        os.path.join(output_dir, f"{dump_name}.yaml"),
                    )

                except Exception as e:
                    logging.error(
                        f"Failed to create output yaml at {self.data_directory} with error: {e}"
                    )


if __name__ == "__main__":
    fire.Fire(DraftSyllabiYAMLRunner)

sciphi/examples/khan_academy/gen_step_1_draft_syllabi.py
emrgnt-cmplxty/sciphi
# type: ignore
"""
YAML Syllabi to YAML 'Draft' Table of Contents Generator

Description:
    This script processes YAML files and generates a detailed Table of Contents for a new textbook accompanying the course.

Usage:
    Command-line interface example:
        $ python sciphi/examples/library_of_phi/gen_step_3_draft_table_of_contents.py run \
            --input_dir=output_step_2 \
            --provider=openai \
            --model_name=gpt-4-0613 \
            --log_level=DEBUG
Parameters:
    provider (str): 
        The provider to use for LLM.
        Default is 'openai'.
    
    model_name (str): 
        The model to use for LLM.
        Default is 'gpt-4-0613'.

    data_directory (Optional[str]): 
        The directory input / output data is stored.
        If none, defaults to the directory of this script plus '/raw_data'.

    input_rel_dir (str): 
        The folder for the input JSONL files containing the first pass YAML data. 
        Defaults to 'output_step_1'.

    output_rel_dir (str): 
        The directory where the parsed YAML files will be saved. 
        Defaults to 'output_step_2'.

    log_level (str): 
        Logging level for the scraper. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL. 
        Default is 'INFO'.
"""

import logging
import os
from glob import glob

import fire

from sciphi.examples.helpers import get_default_settings_provider
from sciphi.examples.khan_academy.prompts import (
    TABLE_OF_CONTENTS_DRAFT_PROMPT,
)

from sciphi.examples.helpers import (
    prase_yaml_completion,
    save_yaml,
)


class DraftTableOfContentsRunner:
    """Class to run the generation of detailed 'draft' Table of Contents for textbooks based on YAML syllabus."""

    def __init__(
        self,
        provider: str = "openai",
        model_name: str = "gpt-4-0613",
        input_rel_dir: str = "output_step_1",
        output_rel_dir: str = "output_step_2",
        data_directory=None,
        log_level: str = "INFO",
    ):
        self.provider = provider
        self.model_name = model_name
        self.input_rel_dir = input_rel_dir
        self.output_rel_dir = output_rel_dir
        self.data_directory = data_directory
        logging.basicConfig(level=getattr(logging, log_level.upper()))

    def run(self):
        """Main function."""

        # Default data directory to the directory of this script combined with 'raw_data'.
        if not self.data_directory:
            self.data_directory = os.path.join(
                os.path.dirname(os.path.abspath(__file__)), "raw_data"
            )

        # Build an LLM and provider interface
        llm_provider = get_default_settings_provider(
            provider=self.provider, model_name=self.model_name
        )
        input_dir = os.path.join(self.data_directory, self.input_rel_dir)
        output_dir = os.path.join(self.data_directory, self.output_rel_dir)

        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        for yml_file_path in glob(f"{input_dir}/*.yaml"):
            try:
                # get end of name
                yml_file_name = os.path.basename(yml_file_path)
                yml_output_file = os.path.join(output_dir, yml_file_name)
                if os.path.exists(yml_output_file):
                    logging.info(
                        f"Skipping {yml_output_file} because it was already created..."
                    )
                    continue
                logging.info(f"Processing YAML file {yml_file_name}...")

                with open(yml_file_path, "r") as yfile:
                    yml_load = yfile.read()
                    course_name = " ".join(yml_file_name.split("_")[:-1])
                formatted_prompt = TABLE_OF_CONTENTS_DRAFT_PROMPT.format(
                    course_name=course_name, context=yml_load
                )
                logging.debug(
                    f"Passing formatted prompt to LLM: {formatted_prompt}"
                )
                completion = llm_provider.get_completion(formatted_prompt)
                yaml_completion = prase_yaml_completion(
                    {"completion": completion}
                )
                yml_load = yaml.safe_load(yaml_completion)
                save_yaml(yml_load, yml_output_file)

                logging.debug(f"Completion: {completion}")

            except Exception as e:
                logging.error(
                    f"Failed to process file {yml_file_name} with error {e}."
                )
                pass


if __name__ == "__main__":
    fire.Fire(DraftTableOfContentsRunner)

sciphi/examples/khan_academy/gen_step_2_draft_table_of_contents.py
emrgnt-cmplxty/sciphi
# type: ignore
"""
YAML Table of Contents to Textbook Generator

WARNING - This script is very rough and needs significant enhancements to fully productionize
Description:
    This script processes YAML files and generates a detailed Table of Contents for a new textbook accompanying the course.

Usage:
    Command-line interface:
        $ python sciphi/examples/library_of_phi/generate_textbook.py run
Parameters:
    provider (str): 
        The provider to use. Default is 'openai'.
    model (str): 
        The model name to use. Default is 'gpt-3.5-turbo-instruct'.
    parsed_dir (str): 
        Directory containing parsed data. Default is 'raw_data'.
    toc_dir (str): 
        Directory for the table of contents. Default is 'table_of_contents'.
    output_dir (str): 
        Directory for the output. Default is 'output_step_4'.
      (str): 
        Name of the textbook. Default is 'Introduction_to_Deep_Learning'.
    max_related_context_to_sample (int): 
        Maximum context to sample. Default is 2000.
    max_prev_snippet_to_sample (int): 
        Maximum previous snippet to sample. Default is 2000.
    do_wiki (bool): 
        Whether to use Wikipedia. Default is True.
    wiki_server_url (str): 
        Wikipedia server URL. Uses environment variable "WIKI_SERVER_URL" if not provided.
    wiki_username (str): 
        Wikipedia username. Uses environment variable "WIKI_SERVER_USERNAME" if not provided.
    wiki_password (str): 
        Wikipedia password. Uses environment variable "WIKI_SERVER_PASSWORD" if not provided.
    log_level (str): 
        Logging level. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL. Default is 'INFO'.
"""
import os

import fire

from sciphi.core.utils import get_data_dir
from sciphi.examples.library_of_phi.gen_step_5_draft_book import (
    TextbookContentGenerator,
)


class TextbookContentGeneratorSimplified:
    def __init__(self) -> None:
        pass

    def run(
        self,
        provider="openai",
        model_name="gpt-4-0613",
        toc_dir="table_of_contents",
        textbook="Aerodynamics_of_Viscous_Fluids",
        max_related_context_to_sample=2_000,
        max_prev_snippet_to_sample=2_000,
        do_wiki=True,
        wiki_server_url=None,
        wiki_username=None,
        wiki_password=None,
        log_level="INFO",
    ):
        """Runs the generation process to create a textbook from the YAML table of contents."""
        TextbookContentGenerator(
            provider,
            model_name,
            data_dir=os.path.join(get_data_dir(), "library_of_phi"),
            toc_dir=toc_dir,
            output_dir="",
            textbook=textbook,
            max_related_context_to_sample=max_related_context_to_sample,
            max_prev_snippet_to_sample=max_prev_snippet_to_sample,
            do_wiki=do_wiki,
            wiki_server_url=wiki_server_url,
            wiki_username=wiki_username,
            wiki_password=wiki_password,
            log_level=log_level,
        ).run()


if __name__ == "__main__":
    fire.Fire(TextbookContentGeneratorSimplified)

sciphi/examples/khan_academy/generate_textbook.py
emrgnt-cmplxty/sciphi
# type: ignore
import textwrap

SYLLABI_CREATION_PROMPT = textwrap.dedent(
    """
Take a deep breath, then closely examine the configuration that follows. You will be asked to produce a similar configuration as part of this task, so pay close attention to the following YAML format.

```yaml
course:
    Quantum Physics I:
        topics:
            - Basic Features of Quantum Mechanics:
                subtopics:
                    - Linearity
                    - Complex Numbers
                    - Non-deterministic
                    - Superposition
                    - Entanglement
            - Experimental Basis of Quantum Physics:
                subtopics:
                    - Two-slit Experiments
                    - Mach-Zehnder Interferometer
                    - Elitzur-Vaidman Bombs
                    - Photoelectric Effect
                    - Compton Scattering
                    - de Broglie Wavelength
            - Wave Mechanics:
                subtopics:
                    - Galilean Transformation of de Broglie Wavelength
                    - Wave-packets and Group Velocity
                    - Matter Wave for a Particle
                    - Momentum and Position Operators
                    - Schrödinger Equation
            - Interpretation of the Wavefunction:
                subtopics:
                    - Probability Density
                    - Probability Current
                    - Current Conservation
                    - Hermitian Operators
            - Expectation Values and Uncertainty:
                subtopics:
                    - Expectation Values of Operators
                    - Time Evolution of Wave-packets
                    - Fourier Transforms
                    - Parseval Theorem
                    - Uncertainty Relation
            - Quantum Physics in One-dimensional Potentials:
                subtopics:
                    - Stationary States
                    - Boundary Conditions
                    - Particle on a Circle
                    - Infinite Square Well
                    - Finite Square Well
                    - Semiclassical Approximations
                    - Numerical Solution by the Shooting Method
                    - Delta Function Potential
                    - Simple Harmonic Oscillator
                    - Reflection and Transmission Coefficients
                    - Ramsauer Townsend Effect
                    - 1D Scattering and Phase Shifts
                    - Levinson's Theorem
            - Angular Momentum and Central Potentials:
                subtopics:
                    - Resonances and Breit-Wigner Distribution
                    - Central Potentials
                    - Algebra of Angular Momentum
                    - Legendre Polynomials
                    - Hydrogen Atom
                    - Energy Levels Diagram
                    - Virial Theorem
                    - Circular Orbits and Eccentricity
                    - Discovery of Spin
```

Given the following context, deduce the new configuration entry for the course "{course_name}". Be sure to fill in the appropriate topics and subtopics.

```md
{context}
```

Notes:
- Filter out any erroneous content like "QUIZ" or "TEST" that might appear in the syllabus. 
- Remove phrases like "UNIT" or "CHAPTER" that signal new sections.
- Attempt to match the length and specificity of the above example; add your own context if necessary to accomplish this or remove extra context.
- You should target **5-10 main topics with 6-10 subtopics each**.
- It should be valid YAML, so be sure to indent properly. Assume that the output of your response will be parsed by a YAML parser.

### Response:
"""
)


TABLE_OF_CONTENTS_DRAFT_PROMPT = """
### Instructions:
Below the course syllabus for {course_name} is shown as a yaml snippet.

```yaml
{context}
```

Pretend you are a prolific author, your task is to translate the shown syllabus into a very detailed Table of Contents for a new textbook that is meant to accompany the course.
Be sure to pick a fitting name for the textbook. Add additional topics so that the table of contents provides much more information than the original syllabus.
Return your final result in an easily parseable yaml format, your final yaml must match the above with course -> textbook, and topics -> chapters, subtopics -> sections, and an additional new key: subsubtopics -> subsections.

### Response:
"""


BOOK_FOREWARD_PROMPT = """
### Instructions:
You are a writing a book titled "{title}". You are currently writing the foreward for the book:

#  Title: {title}


To assist you in this task, you have been provided the context bellow:

### Related Context
```
{related_context}
```

Notes:
- The book is being written in the popular Markdown format.
- The context may be truncated and is meant only to provide a starting point. Feel free to expand on it or take the response in any direction that fits the prompt, but keep it in a voice that is appropriate for an advanced undergraduate course at MIT.
- Avoid making any factual claims or opinions without proper citations or context to support them, stick to the proposed context.
- Begin your response with `## Foreward`

### Response:
"""


TABLE_OF_CONTENTS_CLEAN_PROMPT = """
### Instructions:
Below is a table of contents for the textbook "Aerodynamics of Viscous Fluids: A Comprehensive Study". This table of contents has a specified format consisting of a "textbook" key, followed by the textbook title as a key. Nested under the textbook title are the keys for chapters, sections, and subsections. The table is meant to follow a yaml format, but spacing has been removed to save space.

textbook: 
"Aerodynamics of Viscous Fluids: A Comprehensive Study":

chapters:
- Chapter 1: Fundamental Theorem of Kinematics:
sections:
- Section: 1.1 Convection:
subsections:
- 1.1a Introduction to Convection
- 1.1b Boundary Layer Convection
- 1.1c Applications of Convection

- Section: 1.2 Vorticity:
subsections:
- 1.2a Definition of Vorticity
- 1.2b Vorticity Transport Equation
- 1.2c Vorticity Generation

- Section: 1.3 Strain:
subsections:
- 1.3a Introduction to Strain
- 1.3b Strain Rate Tensor
- 1.3c Rate of Deformation Tensor

- Chapter 2: Eulerian vs. Lagrangian Description:
sections:
- Section: 2.1 Convection Relations:
subsections:
- 2.1a Eulerian and Lagrangian Descriptions
- 2.1b Material Derivative
- 2.1c Applications and Differences

- Section: 2.2 Frame of Reference:
subsections:
- 2.2a Fixed Frame
- 2.2b Moving Frame
- 2.2c Advantages and Limitations

- Section: 2.3 Mathematical Representations:
subsections:
- 2.3a Transformation Equations
- 2.3b Jacobian Matrices
- 2.3c Use in Kinematics

- Chapter 3: Conservation Laws:
sections:
- Section: 3.1 Conservation of Mass:
subsections:
- 3.1a Continuity Equation
- 3.1b Incompressible Flow
- 3.1c Compressible Flow
- Section: 3.2 Conservation of Momentum:
subsections:
- 3.2a Navier-Stokes Equations
- 3.2b Reynolds Transport Theorem
- 3.2c Pressure and Viscous Forces
- Section: 3.3 Stress Tensor:
subsections:
- 3.3a Definition of Stress Tensor
- 3.3b Viscous and Pressure Stresses
- 3.3c Reynolds Stress Tensor

- Chapter 4: Viscosity and Newtonian Fluids:
sections:
- Section: 4.1 Vorticity and Circulation:
subsections:
- 4.1a Vorticity and Circulation Equations
- 4.1b Kelvin's Circulation Theorem
- 4.1c Vorticity Distribution
- 4.1d Circulation and Vorticity Relations
- Section: 4.2 Newtonian vs. Non-Newtonian Fluids:
subsections:
- 4.2a Definition and Characteristics
- 4.2b Models of Non-Newtonian Fluids
- 4.2c Applications and Differences
- Section: 4.3 Shear and Bulk Viscosity:
subsections:
- 4.3a Definitions and Differences
- 4.3b Measurement Techniques
- 4.3c Importance in Fluid Mechanics

- Chapter 5: Navier-Stokes Equations:
sections:
- Section: 5.1 Physical Parameters:
subsections:
- 5.1a Dimensionless Parameters
- 5.1b Reynolds Number
- 5.1c Mach Number
- Section: 5.2 Dynamic Similarity:
subsections:
- 5.2a Scaling Laws
- 5.2b Model Testing
- 5.2c Similarity Criteria

- Chapter 6: Dimensional Analysis:
sections:
- Section: 6.1 Dominant Balance:
subsections:
- 6.1a Balance of Forces
- 6.1b Balance of Stresses
- Section: 6.2 Viscous Flow Classification:
subsections:
- 6.2a Laminar Flow
- 6.2b Turbulent Flow
- Section: 6.3 Buckingham's Pi Theorem:
subsections:
- 6.3a Introduction and Theory
- 6.3b Application in Fluid Mechanics
- 6.3c Examples and Case Studies

- Chapter 7: Thin Shear Layer Equations:
sections:
- Section: 7.1 TSL Coordinates:
subsections:
- 7.1a Definition of TSL Coordinates
- 7.1b TSL Navier-Stokes Equations
- Section: 7.2 Boundary Conditions:
subsections:
- 7.2a No-Slip Condition
- 7.2b Impermeability Condition
- 7.2c Kutta Condition
- Section: 7.3 Shear Layer Categories:
subsections:
- 7.3a Blasius Shear Layer
- 7.3b Falkner-Skan Shear Layer
- 7.3c Displacement Shear Layer

- Chapter 8: Local Scaling:
sections:
- Section: 8.1 Falkner-Skan Flows:
subsections:
- 8.1a Introduction to Falkner-Skan Flows
- 8.1b Boundary Layer Equations
- 8.1c Solution Techniques

- Section: 8.2 Scaling Laws in Fluid Mechanics:
subsections:
- 8.2a Basic Principles of Scaling
- 8.2b Applications in Different Flow Regimes
- 8.2c Limitations and Challenges

- Section: 8.3 Similarity Solutions:
subsections:
- 8.3a Introduction to Similarity Solutions
- 8.3b Methods to Achieve Similarity
- 8.3c Examples in Fluid Dynamics

- Chapter 9: ODE's, PDE's, and Boundary Conditions:
sections:
- Section: 9.1 Well-Posedness:
subsections:
- 9.1a Definition of Well-Posedness
- 9.1b Initial and Boundary Conditions
- Section: 9.2 Partial Differential Equations:
subsections:
- 9.2a Classification of PDEs
- 9.2b Elliptic, Parabolic, and Hyperbolic Equations
- Section: 9.3 Boundary Conditions:
subsections:
- 9.3a Dirichlet Boundary Condition
- 9.3b Neumann Boundary Condition
- 9.3c Mixed Boundary Condition

- Chapter 10: Numerical Methods for ODE's:
sections:
- Section: 10.1 Discretization:
subsections:
- 10.1a Finite Difference Method
- 10.1b Finite Element Method
- 10.1c Finite Volume Method
- Section: 10.2 Stability:
subsections:
- 10.2a Von Neumann Stability Analysis
- 10.2b CFL Condition
- 10.2c Implicit vs. Explicit Methods

- Chapter 11: Integral Methods:
sections:
- Section: 11.1 Integral Momentum Equation:
subsections:
- 11.1a Integral Boundary Layer Equations
- 11.1b Blasius Solution
- 11.1c Falkner-Skan Solution
- Section: 11.2 Thwaites' Method:
subsections:
- 11.2a Introduction to Thwaites' Method
- 11.2b Boundary Layer Growth
- 11.2c Thwaites' Laminar Boundary Layer Solution

- Chapter 12: Dissipation Methods:
sections:
- Section: 12.1 Integral Kinetic Energy Equation:
subsections:
- 12.1a Turbulent Dissipation
- 12.1b Shear Stress Distribution
- 12.1c Integral Energy Equation

- Section: 12.2 Turbulent Dissipation Mechanisms:
subsections:
- 12.2a Basics of Turbulence and Eddies
- 12.2b Cascade Mechanism
- 12.2c Effects on Boundary Layer

- Section: 12.3 Experimental and Numerical Techniques:
subsections:
- 12.3a Techniques for Measuring Dissipation
- 12.3b Numerical Methods and Challenges
- 12.3c Comparison of Experimental and Numerical Results

- Chapter 13: Asymptotic Perturbation Theory:
sections:
- Section: 13.1 Higher-Order Effects:
subsections:
- 13.1a Introduction to Perturbation Theory
- 13.1b Asymptotic Expansion
- 13.1c Higher-Order Corrections
- Section: 13.2 Boundary Layer Perturbation:
subsections:
- 13.2a Introduction to Boundary Layer Perturbation
- 13.2b Application in Viscous Flows
- 13.2c Challenges and Limitations
- Section: 13.3 Nonlinear Perturbations:
subsections:
- 13.3a Introduction to Nonlinear Perturbations
- 13.3b Application in Fluid Dynamics
- 13.3c Advanced Methods and Techniques

- Chapter 14: 2D Interaction Models:
sections:
- Section: 14.1 Displacement Body:
subsections:
- 14.1a Flow Past a Cylinder
- 14.1b Flow Past an Airfoil
- Section: 14.2 Transpiration:
subsections:
- 14.2a Transpiration Boundary Condition
- 14.2b Transpiration Effects on Flow Field
- Section: 14.3 Form Drag:
subsections:
- 14.3a Definition of Form Drag
- 14.3b Pressure Drag
- 14.3c Friction Drag
- Section: 14.4 Stall Mechanisms:
subsections:
- 14.4a Boundary Layer Separation
- 14.4b Flow Reattachment
- 14.4c Dynamic Stall

- Chapter 15: IBLT Solution Techniques:
sections:
- Section: 15.1 Iteration Stability:
subsections:
- 15.1a Stability Analysis
- 15.1b Convergence Criteria
- 15.1c Solution Techniques
- Section: 15.2 Fully-Coupled Iteration:
subsections:
- 15.2a Introduction to Fully-Coupled Iteration
- 15.2b Fluid-Structure Interaction
- 15.2c Aeroelasticity
- Section: 15.3 3-D IBLT:
subsections:
- 15.3a Three-Dimensional Boundary Layers
- 15.3b Transition Mechanisms in 3D Flows

- Chapter 16: Small-Perturbation Theory:
sections:
- Section: 16.1 Orr-Sommerfeld Equation:
subsections:
- 16.1a Definition of Orr-Sommerfeld Equation
- 16.1b Linear Stability Analysis
- 16.1c Growth Rates and Frequencies
- Section: 16.2 Linear Instabilities:
subsections:
- 16.2a Introduction to Linear Instabilities
- 16.2b Applications in Viscous Flows
- 16.2c Detection and Control Methods
- Section: 16.3 Perturbation in Compressible Flows:
subsections:
- 16.3a Basics of Compressible Flow Perturbations
- 16.3b Challenges in High-Speed Flows
- 16.3c Recent Advances and Developments

- Chapter 17: Boundary Conditions, Homogeneity, Solution Techniques:
sections:
- Section: 17.1 Transition Mechanisms:
subsections:
- 17.1a Laminar-Turbulent Transition
- 17.1b Transition Prediction Methods
- Section: 17.2 Transition Prediction:
subsections:
- 17.2a Stability Analysis
- 17.2b Experimental Methods
- 17.2c Computational Methods

- Chapter 18: Reynolds Averaging:
sections:
- Section: 18.1 Prandtl's Analogy:
subsections:
- 18.1a Reynolds-Averaged Navier-Stokes Equations
- 18.1b Turbulent Stresses
- 18.1c Reynolds Stress Modeling

- Chapter 19: Turbulent BL Structure:
sections:
- Section: 19.1 Wake:
subsections:
- 19.1a Wake Formation
- 19.1b Wake Characteristics
- Section: 19.2 Wall Layers:
subsections:
- 19.2a Viscous Sublayer
- 19.2b Buffer Layer
- 19.2c Overlap Layer
- Section: 19.3 Inner, Outer Variables:
subsections:
- 19.3a Velocity and Length Scales
- 19.3b Wall Coordinates
- 19.3c Outer Variables
- Section: 19.4 Effects of Roughness:
subsections:
- 19.4a Roughness Effects on Boundary Layer
- 19.4b Skin Friction and Heat Transfer

- Chapter 20: Equilibrium BL's:
sections:
- Section: 20.1 Clauser Hypothesis:
subsections:
- 20.1a Equilibrium Boundary Layer Concept
- 20.1b Clauser's Equilibrium Assumptions
- Section: 20.2 Dissipation Formulas:
subsections:
- 20.2a Turbulent Energy Dissipation
- 20.2b Dissipation in Equilibrium Boundary Layers
- Section: 20.3 Integral Closure:
subsections:
- 20.3a Reynolds Stress Closure
- 20.3b Eddy Viscosity Models
- 20.3c Mixing Length Models
...

Your task is to inspect the table of contents that is provided below and to produce a new table of contents which is formatted EXACTLY as the example above. 

{table_of_contents}

Further, if the table of contents is less than 20 chapters, or lacking sufficient detail in any given chapter, then your task is to coherently expand the table and re-arrange it so that it is 20+ chapters, with 3-4 sections and 3-4 subsections per chapter, formatted exactly as the example above.

If the table of contents is already in the correct format and complete, reproduce it as-is with the new formatting.


Notes:
- Chapters should be numbered `Chapter 1`, `Chapter 2`, etc.
- Sections should be numbered `Section 1.1`, `Section 1.2`, etc.
- Subsections should be numbered `1.1a`, `1.1b`, etc.
- The table of contents should contain no indentations.
- The table of contents should be 20 chapters or more, and have 3-4 sections and 3-4 subsections per chapter.

### Response:
"""

BOOK_BULK_PROMPT = """
### Instructions:
You are a writing a book titled "{title}". You are currently writing the chapter and section shown below:

#  Title: {title}

## Chapter: {chapter}

### Section: {section}

### Subsection (optional): {subsection}

To assist you in writing the chapter, you have been provided with some related context and recent chapter contents bellow:

### Related Context
```
{related_context}
```

### Last textbook section content:
```
{book_context}
```

Notes:
- The book is being written in the popular Markdown format.
- The context may be truncated and is meant only to provide a starting point. Feel free to expand on it or take the response in any direction that fits the prompt, but keep it in a voice that is appropriate for an advanced undergraduate course at MIT.
- Avoid making any factual claims or opinions without proper citations or context to support them, stick to the proposed context.
- Format ALL math equations with the $ and $$ delimiters to insert math expressions in TeX and LaTeX style syntax. This content is then rendered using the highly popular MathJax library. E.g. write inline math like `$y_j(n)$` and equations like `$$
\\Delta w = ...
$$
- If starting a new section, include `### [Section Title]`
- If starting a new subsection, include `#### [Subsection Title]`
`

### Response:
"""

# Prompt for the LLM
BOOK_CHAPTER_SUMMARY_PROMPT = """
### Instructions:
You are a writing a book titled "{title}". You are tasked with writing a several paragraph CONCLUSION FOR THE CHAPTER shown below:

#  Title: {title}

## Chapter: {chapter}

To assist you in this task, you have been provided the context bellow:

### Previously written chapter:
```
{book_context}
```

Following your authoring of the conclusion, write 5 exercises which align with the context of the chapter. Format each with a header `#### Exercise 1` etc.

Notes:
- The book is being written in the popular Markdown format.
- Avoid making any factual claims or opinions without proper citations or context to support them, stick to the proposed context.
- Format ALL math equations with the $ and $$ delimiters to insert math expressions in TeX and LaTeX style syntax. This content is then rendered using the highly popular MathJax library. E.g. write inline math like `$y_j(n)$` and equations like `$$
\\Delta w = ...
$$
- Start the conclusion the Chapter with a header that reads `### Conclusion`, start the exercises with a header that reads `### Exercises`.

`

### Response:
"""

# Prompt for the LLM
BOOK_CHAPTER_INTRODUCTION_PROMPT = """
### Instructions:
You are a writing a book titled "{title}". You are currently writing a several paragraph introducton for the chapter shown below (avoid going into too much detail):

#  Title: {title}

## Chapter: {chapter}

To assist you in this task, you have been provided the context which will be covered in this chapter:

### Chapter section topics covered:
```
{book_context}
```

Notes:
- The book is being written in the popular Markdown format.
- Avoid making any factual claims or opinions without proper citations or context to support them, stick to the proposed context.
- Format ALL math equations with the $ and $$ delimiters to insert math expressions in TeX and LaTeX style syntax. This content is then rendered using the highly popular MathJax library. E.g. write inline math like `$y_j(n)$` and equations like `$$
\\Delta w = ...
$$
- Include the chapter title at the top of your output, formatted as  `## Chapter: [Title]`,  `### Introduction` below that.
`

### Response:
"""

sciphi/examples/khan_academy/prompts.py
emrgnt-cmplxty/sciphi
"""
KA Course Data Scraper

Description:
    This script is designed to scrape course data from the OpenCourseWare (KA) website.
    It extracts details like course number, title, URL, resource level, page contents, and more.

Usage:
    Command-line interface:
        $ poetry run python sciphi/examples/library_of_phi/raw_data/ka_scraper.py scrape \
            --input_file_name=ka_topics.html \
            --output_file_name=scraped_ka.jsonl \
            --sleep_time=1 \
            --log_level=INFO

Parameters:
    data_directory (Optional[str], default=None): 
        Path to the directory where the input file is located 
        and where the output file will be saved. 
        If not specified, defaults to the directory of this script.

    input_file_name (str, default="ka_topics.html"): 
        Name of the input HTML file containing KA course listings.
    
    output_file_name (str, default="ka_scraped.jsonl"): 
        Name of the output JSONL file where scraped data will be saved.
        
    sleep_time (int, default=0): 
        Time (in seconds) to pause between scraping consecutive courses.
    
    log_level (str, default="INFO"): 
        Logging level for the scraper. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL.
"""
import logging
import os
import re
import time
from typing import Dict, List, Optional, Union

import fire
import requests
from bs4 import BeautifulSoup

from sciphi.writers import JsonlDataWriter

# Constants
BASE_URL = "https://www.khanacademy.org"
USER_AGENT = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
)
HEADERS = {"User-Agent": USER_AGENT}
PAGE_TYPES = [
    "syllabus",
]

logging.basicConfig(level=logging.INFO)


class Scraper:
    """Scraper for KA course data."""

    def __init__(self, sleep_time: int = 0, log_level: str = "INFO"):
        self.sleep_time = sleep_time
        logging.basicConfig(level=getattr(logging, log_level.upper()))

    def scrape(
        self,
        data_directory: Optional[str] = None,
        input_file_name: str = "ka_courses.html",
        output_file_name: str = "ka_scraped.jsonl",
    ):
        """Scrape course data from KA and save to an output file."""

        if not data_directory:
            data_directory = os.path.dirname(os.path.abspath(__file__))

        with open(os.path.join(data_directory, input_file_name)) as f_open:
            html_content = f_open.read()

        soup = BeautifulSoup(html_content, "html.parser")

        # Find all the 'a' tags with 'href' attributes
        courses = soup.find_all("a", href=True)

        # Extract and print the href values and corresponding text
        pass_through = True
        for a_tag in courses:
            href = a_tag["href"]
            text = a_tag.text.strip()  # Remove leading/trailing whitespace

        writer = JsonlDataWriter(
            os.path.join(data_directory, output_file_name)
        )

        course_dictionary = {}

        # This section handles many of the edge cases in the KA course listings
        for course in courses:
            if "/humanities/art-history" == course["href"]:
                continue

            if course["href"].count("/") < 2:
                continue

            if (
                "digital-sat" in course["href"]
                or "/sat/" in course["href"]
                or "prep" in course["href"]
                or "college-careers-more" in course["href"]
                or "partner-content" in course["href"]
            ):
                continue

            course_data = Scraper.extract_course_data(course)
            writer.write([course_data])

            time.sleep(self.sleep_time)

    @staticmethod
    def get_soup(page_url: str) -> Optional[BeautifulSoup]:
        """Retrieve and parse a webpage."""
        try:
            response = requests.get(page_url, headers=HEADERS)
            response.raise_for_status()
            return BeautifulSoup(response.text, "html.parser")
        except requests.RequestException as e:
            logging.error(f"Failed to retrieve {page_url}. Error: {e}")
            return None

    @staticmethod
    def read_page(
        page_url: str,
    ) -> Dict[str, Union[str, List[Dict[str, str]]]]:
        """Read the given page URL and return a dictionary of the extracted data."""
        logging.info(f"Scraping {page_url}")

        ka_link = f"{BASE_URL}{page_url.rstrip('/')}" if page_url else None
        soup = Scraper.get_soup(ka_link)

        units = soup.find_all("a")
        result = {}
        result["topics"] = {}
        current_topic_url = None
        current_topic_key = None

        for unit in units:
            if (
                page_url in unit["href"]
                and "/a/" not in unit["href"]
                and not unit["href"].endswith("/d")
                and "/e/" not in unit["href"]
                and "/quiz/" not in unit["href"]
                and "privacy" not in unit.text.strip().lower()
                and "skills" not in unit.text.strip().lower()
            ):
                if (
                    "unit" in unit.text.strip().lower()
                    or current_topic_url not in unit["href"]
                ):
                    current_topic_url = page_url
                    current_topic_key = unit.text.strip()

                    result["topics"][current_topic_key] = []

                else:
                    if unit.text.strip().lower() == "start course challenge":
                        continue
                    result["topics"][current_topic_key].append(
                        unit.text.strip()
                    )

        # REMOVE all empty topics
        # We remove <= 1 because certain classes in KA only have an introduction
        empty_topics = []
        for topic in result["topics"].keys():
            if len(result["topics"][topic]) <= 1:
                empty_topics.append(topic)

        for topic in empty_topics:
            del result["topics"][topic]

        return result

    @staticmethod
    def _extract_text(
        element: BeautifulSoup, tag: str, class_: str
    ) -> Optional[str]:
        """Extract text from an element with a specific tag and class."""
        found_element = element.find(tag, class_=class_)
        return found_element.get_text(strip=True) if found_element else None

    def _extract_leading_text(soup: BeautifulSoup) -> str:
        """Extract leading text from the soup."""
        main_section = soup.find("main", id="course-content-section")
        if main_section:
            paragraphs = main_section.find_all("p")
            leading_text = " ".join(
                paragraph.get_text(strip=True) for paragraph in paragraphs
            )
            return Scraper._clean_text(leading_text)
        return ""

    def extract_course_data(
        course: BeautifulSoup,
    ) -> Dict[str, Union[str, List[Dict[str, str]]]]:
        """Extract course data from an article tag."""
        logging.debug("Extracting course data...")

        course_title = course.text.strip()
        course_url = course["href"]

        course_data = {
            "course_title": course_title,
            "course_url": course_url,
            "page_contents": {},
        }

        # Only scrape the syllabus page for now
        for page_type in PAGE_TYPES:
            try:
                course_data["page_contents"][page_type] = Scraper.read_page(
                    course_url
                )
            except requests.RequestException:
                pass

        return course_data

    @staticmethod
    def _extract_table_rows(soup: BeautifulSoup) -> List[Dict[str, str]]:
        """Extract table rows from the soup."""
        logging.debug("Extracting table rows...")

        rows = soup.find_all("tr")
        table_data = []

        for row in rows:
            try:
                note_number = Scraper._clean_text(
                    row.find_all("td")[0].get_text(strip=True)
                )
                topic_col = row.find_all("td")[1]
                topic = Scraper._clean_text(
                    topic_col.get_text(strip=True).split("(")[0].strip()
                )
                link = topic_col.find("a")
                note_url = link["href"] if link else None
                table_data.append(
                    {
                        "note_number": note_number,
                        "topic": topic,
                        "note_url": note_url,
                    }
                )
            except IndexError:
                logging.debug(
                    "Skipping row that doesn't match the expected format"
                )
                pass  # Skip rows that don't match the expected format

        return table_data

    @staticmethod
    def _clean_text(text: str) -> str:
        """Clean the given text."""
        text = text.replace("\xa0", " ")
        return re.sub(r"\s+", " ", text).strip()

    @staticmethod
    def _get_topics_and_depths(soup: BeautifulSoup) -> Dict[str, int]:
        """Extract topics and their depths from the soup."""
        topics = soup.select("span.topic-text-wrapper > a.course-info-topic")
        return {
            topic.get_text(strip=True): len(
                topic.find_parent("li").find_parents("ul")
            )
            - 1
            for topic in topics
        }


def main():
    """Main function."""
    fire.Fire(Scraper)


if __name__ == "__main__":
    main()

sciphi/examples/khan_academy/raw_data/ka_scraper.py
emrgnt-cmplxty/sciphi
# type: ignore
"""
MIT OCW Course Data Scraper to syllyabi 'Draft' YAML Generator

Description:
    This script is designed to scrape course data from the MIT OpenCourseWare (OCW) website and 
    generate input YAML files with the help of an LLM.

Usage:
    Command-line interface:
        $ python sciphi/examples/library_of_phi/gen_step_1_draft_syllabi.py run \
            --output_rel_dir=my_output_directory \
            --input_jsonl_filename=my_input_file.jsonl \
            --log_level=DEBUG
Parameters:
    
    provider (str):
        The provider to use for LLM completions.
        Default is 'openai'.
    
    model_name (str):
        The model to use for LLM completions.
        Default is 'gpt-4-0613'.

    data_directory (Optional[str]): 
        The directory the input and output data is to be stored.
        If none, defaults to the directory of this script plus '/raw_data'.

    output_rel_dir (str): 
        The relative directory within the data directory where the generated YAML files will be saved. 
        Default value is 'output_step_1'.

    input_jsonl_filename (str): 
        The name of the input jsonl file containing course data scraped from MIT OCW.
        Default value is 'output_step_1'.

    log_level (str): 
        Logging level for the scraper. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL. 
        Default is 'INFO'.
"""

import json
import logging
import os
from glob import glob
from typing import Optional, Set

import fire
import yaml

from sciphi.examples.helpers import get_default_settings_provider
from sciphi.examples.library_of_phi.prompts import SYLLABI_CREATION_PROMPT


def extract_data_from_record(record: dict[str, str]) -> tuple[dict, str]:
    """Extract and organize data from a given record."""
    context = f"### Course Number:\n{record['course_number']}\n"
    context += f"### Course Name:\n{record['course_title']}\n"
    context += f"### Resource Level: {record['resource_level']}\n"
    topics = {}
    page_contents = record["page_contents"]

    for key in page_contents.keys():
        page = page_contents[key]
        context += f"## Page:\n{key}\n"
        if key != "syllabus":
            context += f"Text:\n{page['leading_text'][:500]}\n"
        else:
            topics = page["topics"]
        table = "\n".join([ele["topic"] for ele in page["table_rows"]])
        context += f"Information:\n{table}\n"

    return topics, context


def get_mapped_topics(topics: str) -> dict[str, str]:
    """Get mapped topics based on depth."""
    mapped_topics = {}
    for topic in topics:
        if topics[topic] == 0:
            mapped_topics["category"] = topic
        elif topics[topic] == 1:
            mapped_topics["field"] = topic
        elif topics[topic] == 2:
            mapped_topics["subfield"] = topic
            break
    return mapped_topics


def get_observed_files(output_dir: str) -> Set[str]:
    """Get set of previously observed files."""
    observed_files = set([])
    for yaml_file in glob(f"{output_dir}/**/*.yaml"):
        with open(yaml_file, "r") as existing_file:
            yaml_load = yaml.safe_load(existing_file)
            if "course" in yaml_load:
                for course_name in yaml_load["course"]:
                    _, discipline, field = yaml_file.split("/")
                    field = field.replace(".yaml", "")
                    subfield = yaml_load["course"][course_name]["subfield"]
                    observed_files.add(
                        f"field_{field}_subfield_{subfield}_course_name_{course_name.replace(' ','_')}"
                    )
    for yaml_dumps in glob(f"{output_dir}/*.yaml"):
        observed_files.add(yaml_dumps.split("/")[-1].replace(".yaml", ""))
    return observed_files


def quoted_presenter(dumper, data):
    """Define a custom representer for string scalars that applies quotes."""
    return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"')


class DraftSyllabiYAMLRunner:
    """Runs the generation process for draft syllabi YAMLs."""

    def __init__(
        self,
        provider: str = "openai",
        model_name: str = "gpt-4-0613",
        data_directory: Optional[str] = None,
        output_rel_dir: str = "output_step_1",
        input_jsonl_filename: str = "ocw_scraped.jsonl",
        prompt: str = SYLLABI_CREATION_PROMPT,
        log_level: str = "INFO",
    ):
        self.provider = provider
        self.model_name = model_name
        self.data_directory = data_directory
        self.output_rel_dir = output_rel_dir
        self.input_jsonl_filename = input_jsonl_filename
        self.prompt = prompt
        logging.basicConfig(level=getattr(logging, log_level.upper()))

    def run(self) -> None:
        """Run the draft YAML generation process."""
        yaml.add_representer(str, quoted_presenter)

        llm_provider = get_default_settings_provider(
            provider=self.provider, model_name=self.model_name
        )
        if not self.data_directory:
            file_path = os.path.dirname(os.path.abspath(__file__))
            self.data_directory = os.path.join(file_path, "raw_data")

        output_dir = os.path.join(self.data_directory, self.output_rel_dir)
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
        logging.info(f"Saving data to output directory = {output_dir}")

        # The input file path = data directory + input file name
        input_file_path = os.path.join(
            self.data_directory, self.input_jsonl_filename
        )

        with open(input_file_path, "r") as file:
            for line in file:
                try:
                    record = json.loads(line)
                    topics, context = extract_data_from_record(record)
                    mapped_topics = get_mapped_topics(topics)

                    if len(mapped_topics) != 3:
                        continue

                    category = (
                        mapped_topics["category"].lower().replace(" ", "_")
                    )
                    field = mapped_topics["field"].lower().replace(" ", "_")
                    subfield = (
                        mapped_topics["subfield"].lower().replace(" ", "_")
                    )
                    course_name = record["course_title"]

                    dump_name = f"field_{field}_subfield_{subfield}_course_name_{course_name.replace(' ','_')}"
                    observed_files = get_observed_files(output_dir)

                    if dump_name in observed_files:
                        logging.info(f"Skipping output file {dump_name}....")
                        continue

                    logging.info(f"Saving output file named {dump_name}.")
                    formatted_prompt = self.prompt.format(
                        course_name=course_name, context=context
                    )
                    completion = llm_provider.get_completion(formatted_prompt)
                    data_to_save = {
                        "completion": completion,
                        "discipline": category,
                        "resource_level": record["resource_level"],
                        "field": field,
                        "subfield": subfield,
                        "course_name": course_name,
                    }

                    with open(
                        os.path.join(output_dir, f"{dump_name}.yaml"),
                        "w",
                    ) as f:
                        yaml.dump(data_to_save, f, default_flow_style=False)

                except Exception as e:
                    logging.error(
                        f"Failed to create output yaml at {self.data_directory} with error: {e}"
                    )


if __name__ == "__main__":
    fire.Fire(DraftSyllabiYAMLRunner)

sciphi/examples/library_of_phi/gen_step_1_draft_syllabi.py
emrgnt-cmplxty/sciphi
"""
Syllabi YAML 'draft' to Clean Syllabi YAML Generator

Description:
    This script is designed to scrape course data from the MIT OpenCourseWare (OCW) website and 
    generate YAML files suitable for LLM (Language Learning Model).

Usage:
    Command-line interface:
        $ python sciphi/examples/library_of_phi/gen_step_2_clean_syllabi.py run \
            --output_dir=my_output_directory \
            --input_filename=my_input_file.jsonl \
            --log_level=DEBUG

Parameters:
    data_directory (Optional[str]): 
        The directory input / output data is stored.
        If none, defaults to the directory of this script plus '/raw_data'.

    input_rel_dir (str): 
        The folder for the input JSONL files containing the first pass YAML data. 
        Defaults to 'output_step_1'.

    output_rel_dir (str): 
        The directory where the parsed YAML files will be saved. 
        Defaults to 'output_step_2'.

    log_level (str): 
        Logging level for the scraper. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL. 
        Default is 'INFO'.
"""

import glob
import logging
import os

import fire
import yaml

from sciphi.examples.helpers import (
    load_yaml_file,
    prase_yaml_completion,
    save_yaml,
)


class CleanSyllabiYAMLRunner:
    """Runs the generation process to clean drafted syllabi YAMLs."""

    def __init__(
        self,
        data_directory=None,
        input_rel_dir="output_step_1",
        output_rel_dir="output_step_2",
        log_level="INFO",
    ):
        self.data_directory = data_directory
        self.input_rel_dir = input_rel_dir
        self.output_rel_dir = output_rel_dir
        self.log_level = log_level

    def run(self):
        # Set up logging
        logging.basicConfig(level=self.log_level.upper())
        logger = logging.getLogger(__name__)

        # Default data directory to the directory of this script combined with 'raw_data'.
        if not self.data_directory:
            self.data_directory = os.path.join(
                os.path.dirname(os.path.abspath(__file__)), "raw_data"
            )

        input_yaml_dir = os.path.join(self.data_directory, self.input_rel_dir)
        # Validate input directory exists
        if not os.path.exists(input_yaml_dir):
            logger.error(f"Input directory '{input_yaml_dir}' does not exist")
            raise ValueError(
                f"Input directory '{input_yaml_dir}' does not exist"
            )

        output_yaml_dir = os.path.join(
            self.data_directory, self.output_rel_dir
        )
        if not os.path.exists(output_yaml_dir):
            logger.info(
                f"Output directory '{output_yaml_dir}' does not exist, creating it..."
            )
            os.makedirs(output_yaml_dir)

        counter = 0
        for yml_file_path in glob.glob(os.path.join(input_yaml_dir, "*.yaml")):
            try:
                yml_content = load_yaml_file(yml_file_path)
                parsed_yml_str = prase_yaml_completion(yml_content)
                yml_load = yaml.safe_load(parsed_yml_str)
                yml_file_name = os.path.basename(yml_file_path)
                save_yaml(
                    yml_load, os.path.join(output_yaml_dir, yml_file_name)
                )
                counter += 1
            except (ValueError, yaml.YAMLError) as exc:
                logger.error(
                    f"Error parsing YAML string in {yml_file_path}: {exc}"
                )
                continue

        logger.info(f"Successfully parsed {counter} files")


if __name__ == "__main__":
    fire.Fire(CleanSyllabiYAMLRunner)

sciphi/examples/library_of_phi/gen_step_2_clean_syllabi.py
emrgnt-cmplxty/sciphi
# type: ignore
"""
YAML Syllabi to YAML 'Draft' Table of Contents Generator

Description:
    This script processes YAML files and generates a detailed Table of Contents for a new textbook accompanying the course.

Usage:
    Command-line interface example:
        $ python sciphi/examples/library_of_phi/gen_step_3_draft_table_of_contents.py run \
            --input_dir=output_step_2 \
            --provider=openai \
            --model_name=gpt-4-0613 \
            --log_level=DEBUG
Parameters:
    provider (str): 
        The provider to use for LLM.
        Default is 'openai'.
    
    model_name (str): 
        The model to use for LLM.
        Default is 'gpt-4-0613'.

    data_directory (Optional[str]): 
        The directory input / output data is stored.
        If none, defaults to the directory of this script plus '/raw_data'.

    input_rel_dir (str): 
        The folder for the input JSONL files containing the first pass YAML data. 
        Defaults to 'output_step_1'.

    output_rel_dir (str): 
        The directory where the parsed YAML files will be saved. 
        Defaults to 'output_step_2'.

    log_level (str): 
        Logging level for the scraper. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL. 
        Default is 'INFO'.
"""

import logging
import os
from glob import glob

import fire

from sciphi.examples.helpers import get_default_settings_provider
from sciphi.examples.library_of_phi.prompts import (
    TABLE_OF_CONTENTS_DRAFT_PROMPT,
)


class DraftTableOfContentsRunner:
    """Class to run the generation of detailed 'draft' Table of Contents for textbooks based on YAML syllabus."""

    def __init__(
        self,
        provider: str = "openai",
        model_name: str = "gpt-4-0613",
        input_rel_dir: str = "output_step_2",
        output_rel_dir: str = "output_step_3",
        data_directory=None,
        log_level: str = "INFO",
    ):
        self.provider = provider
        self.model_name = model_name
        self.input_rel_dir = input_rel_dir
        self.output_rel_dir = output_rel_dir
        self.data_directory = data_directory
        logging.basicConfig(level=getattr(logging, log_level.upper()))

    def run(self):
        """Main function."""

        # Default data directory to the directory of this script combined with 'raw_data'.
        if not self.data_directory:
            self.data_directory = os.path.join(
                os.path.dirname(os.path.abspath(__file__)), "raw_data"
            )

        # Build an LLM and provider interface
        llm_provider = get_default_settings_provider(
            provider=self.provider, model_name=self.model_name
        )
        input_dir = os.path.join(self.data_directory, self.input_rel_dir)
        output_dir = os.path.join(self.data_directory, self.output_rel_dir)

        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        for yml_file_path in glob(f"{input_dir}/*.yaml"):
            try:
                # get end of name
                yml_file_name = os.path.basename(yml_file_path)
                yml_output_file = os.path.join(output_dir, yml_file_name)
                if os.path.exists(yml_output_file):
                    logging.info(
                        f"Skipping {yml_output_file} because it was already created..."
                    )
                    continue
                logging.info(f"Processing YAML file {yml_file_name}...")

                with open(yml_file_path, "r") as yfile:
                    yml_load = yfile.read()
                    course_name = " ".join(
                        yml_file_name.split("name_")[1].split("-")
                    )
                formatted_prompt = TABLE_OF_CONTENTS_DRAFT_PROMPT.format(
                    course_name=course_name, context=yml_load
                )
                logging.debug(
                    f"Passing formatted prompt to LLM: {formatted_prompt}"
                )
                completion = llm_provider.get_completion(formatted_prompt)
                logging.debug(f"Completion: {completion}")
                with open(yml_output_file, "w") as yfile:
                    yfile.write(completion)
            except Exception as e:
                logging.error(
                    f"Failed to process file {yml_file_name} with error {e}."
                )
                pass


if __name__ == "__main__":
    fire.Fire(DraftTableOfContentsRunner)

sciphi/examples/library_of_phi/gen_step_3_draft_table_of_contents.py
emrgnt-cmplxty/sciphi
# type: ignore
"""
YAML Syllabi to YAML 'Draft' Table of Contents Generator

Description:
    This script processes YAML files and generates a detailed Table of Contents for a new textbook accompanying the course.

Usage:
    Command-line interface example:
        $ python sciphi/examples/library_of_phi/gen_step_4_clean_table_of_contents.py run \
            --input_dir=output_step_3 \
            --provider=openai \
            --model_name=gpt-4-0613 \
            --log_level=DEBUG
Parameters:
    provider (str): 
        The provider to use for LLM.
        Default is 'openai'.
    
    model_name (str): 
        The model to use for LLM.
        Default is 'gpt-4-0613'.

    data_directory (Optional[str]): 
        The directory input / output data is stored.
        If none, defaults to the directory of this script plus '/raw_data'.

    input_rel_dir (str): 
        The folder for the input JSONL files containing the first pass YAML data. 
        Defaults to 'output_step_1'.

    output_rel_dir (str): 
        The directory where the parsed YAML files will be saved. 
        Defaults to 'output_step_2'.

    log_level (str): 
        Logging level for the scraper. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL. 
        Default is 'INFO'.
    
    num_processes (int):
        Number of processes to use. Defaults to the number of CPUs available.
"""

import logging
import os
import pathlib
from concurrent.futures import ThreadPoolExecutor
from glob import glob

import fire
import yaml

from sciphi.examples.helpers import (
    get_default_settings_provider,
    traverse_config,
    with_retry,
)
from sciphi.examples.library_of_phi.prompts import (
    TABLE_OF_CONTENTS_CLEAN_PROMPT,
)


def ensure_directory_exists(directory_path: str):
    """Ensure that the directory exists. If not, create it."""
    if not os.path.exists(directory_path):
        os.makedirs(directory_path)


def get_field_and_subfield_from_filename(filename: str) -> tuple:
    """Extract field and subfield from the given filename."""
    field = filename.split("_subfield_")[0][6:]
    subfield = filename.split("subfield_")[1].split("_course_name")[0]
    return field, subfield


# Utility function to construct the config from text
def construct_config(text: str) -> dict:
    lines = text.strip().split("\n")

    # Initial structure
    toc = {}
    current_chapter = None
    current_section = None

    for i, line in enumerate(lines):
        stripped = line.strip()

        if stripped.startswith("textbook:"):
            textbook_title = lines[i + 1].strip().strip('"')
            toc["textbook"] = {textbook_title: {"chapters": []}}
        elif stripped.startswith("- Chapter"):
            current_chapter = {stripped.strip(): {"sections": []}}
            toc["textbook"][textbook_title]["chapters"].append(current_chapter)
        elif stripped.startswith("- Section:"):
            current_section = {stripped.strip(): {"subsections": []}}
            if current_chapter:
                current_chapter_key = list(current_chapter.keys())[0]
                current_chapter[current_chapter_key]["sections"].append(
                    current_section
                )
        elif stripped.startswith("- ") and current_section:
            current_section_key = list(current_section.keys())[0]
            current_section[current_section_key]["subsections"].append(
                stripped[2:].strip()
            )
    return toc


class CleanTableOfContentsRunner:
    """Class to run the cleaning of detailed Table of Contents."""

    def __init__(
        self,
        provider: str = "openai",
        model_name: str = "gpt-3.5-turbo-16k-0613",
        data_directory=None,
        input_rel_dir: str = "output_step_3",
        output_rel_dir: str = "output_step_4",
        log_level: str = "INFO",
        num_processes: int = None,
    ):
        self.input_rel_dir = input_rel_dir
        self.output_rel_dir = output_rel_dir
        self.data_directory = data_directory
        self.provider = provider
        self.model_name = model_name
        # Build an LLM and provider interface
        self.llm_provider = get_default_settings_provider(
            provider=self.provider, model_name=self.model_name
        )

        logging.basicConfig(level=getattr(logging, log_level.upper()))
        self.num_processes = num_processes or os.cpu_count()
        logging.info("Running with {self.num_processes} processes")

        # Default data directory to the directory of this script combined with 'raw_data'.
        if not self.data_directory:
            self.data_directory = os.path.join(
                os.path.dirname(os.path.abspath(__file__)), "raw_data"
            )

        self.input_dir = os.path.join(self.data_directory, self.input_rel_dir)
        self.output_dir = os.path.join(
            self.data_directory, self.output_rel_dir
        )

        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)

    def save_config_to_file(self, yml_config, yml_file_path):
        """Save the YAML config to the appropriate directory structure."""
        name = os.path.basename(yml_file_path)
        field, subfield = get_field_and_subfield_from_filename(name)

        field_dir = os.path.join(self.output_dir, field)
        subfield_dir = os.path.join(field_dir, subfield)

        ensure_directory_exists(field_dir)
        ensure_directory_exists(subfield_dir)

        output_filepath = os.path.join(subfield_dir, name)
        logging.info(f"Saving config to {output_filepath}")
        if os.path.exists(output_filepath):
            logging.info(f"File {output_filepath} already exists. Skipping...")
            return

        with open(output_filepath, "w") as f:
            yaml.dump(yml_config, f, sort_keys=False)

        logging.info(f"Saved config to {output_filepath}")

    def process_file(self, yml_file_path):
        logging.info(f"Processing file at {yml_file_path}")

        name = os.path.basename(yml_file_path)
        field, subfield = get_field_and_subfield_from_filename(name)

        field_dir = os.path.join(self.output_dir, field)
        subfield_dir = os.path.join(field_dir, subfield)
        output_filepath = os.path.join(subfield_dir, name)

        if os.path.exists(output_filepath):
            logging.info(f"File {output_filepath} already exists. Skipping...")
            return

        # Ensure directories exist
        ensure_directory_exists(field_dir)
        ensure_directory_exists(subfield_dir)

        table_of_contents = pathlib.Path(yml_file_path).read_text()
        prompt = TABLE_OF_CONTENTS_CLEAN_PROMPT.format(
            table_of_contents=table_of_contents
        )
        completion = with_retry(
            lambda: self.llm_provider.get_completion(prompt)
        )

        # Save the raw completion
        with open(output_filepath.replace(".yaml", "_raw.txt"), "w") as f:
            f.write(completion)
        logging.info(f"Completion for {output_filepath} = {completion}")

        try:
            yml_config = construct_config(completion)
            traverse_config(yml_config)
            self.save_config_to_file(yml_config, output_filepath)
        except Exception as e:
            logging.error(
                f"Failed to traverse config at {output_filepath} due to {e}"
            )

    def run(self):
        with ThreadPoolExecutor(max_workers=self.num_processes) as executor:
            list(
                executor.map(
                    self.process_file, glob(f"{self.input_dir}/*.yaml")
                )
            )


if __name__ == "__main__":
    fire.Fire(CleanTableOfContentsRunner)

sciphi/examples/library_of_phi/gen_step_4_clean_table_of_contents.py
emrgnt-cmplxty/sciphi
# type: ignore
"""
YAML Table of Contents to Textbook Generator

WARNING - This script is very rough and needs significant enhancements to fully productionize
Description:
    This script processes YAML files and generates a detailed Table of Contents for a new textbook accompanying the course.

Usage:
    Command-line interface:
        $ python sciphi/examples/library_of_phi/gen_step_5_draft_book.py run \
            --input_dir=output_step_4  \
            --provider=openai \
            --model_name=gpt-4-0613 \
            --log_level=DEBUG
Parameters:
    provider (str): 
        The provider to use. Default is 'openai'.
    model_name (str): 
        The model_name to use. Default is 'gpt-4-0613'.
    toc_dir (str): 
        Directory for the table of contents. Default is 'output_step_4'.
    output_dir (str): 
        Directory for the output. Default is 'output_step_5'.
      (str): 
        Name of the textbook. Default is 'Introduction_to_Deep_Learning'.
    data_dir (Optional[str]):
        Directory where data is to be read from, defaults to `None`.
        When `None`, defaults to the `[Local Script Directory]/raw_data`
    max_related_context_to_sample (int): 
        Maximum context to sample. Default is 2000.
    max_prev_snippet_to_sample (int): 
        Maximum previous snippet to sample. Default is 2000.
    do_wiki (bool): 
        Whether to use Wikipedia. Default is True.
    wiki_server_url (str): 
        Wikipedia server URL. Uses environment variable "WIKI_SERVER_URL" if not provided.
    wiki_username (str): 
        Wikipedia username. Uses environment variable "WIKI_SERVER_USERNAME" if not provided.
    wiki_password (str): 
        Wikipedia password. Uses environment variable "WIKI_SERVER_PASSWORD" if not provided.
    log_level (str): 
        Logging level. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL. Default is 'INFO'.
    num_threads (int):
        Number of threads to use. Defaults to the number of CPUs available.
"""
import glob
import logging
import multiprocessing
import os
from typing import Tuple

import fire

from sciphi.examples.helpers import (
    get_default_settings_provider,
    load_yaml_file,
    traverse_config,
    wiki_search_api,
    with_retry,
)
from sciphi.examples.library_of_phi.prompts import (
    BOOK_BULK_PROMPT,
    BOOK_CHAPTER_INTRODUCTION_PROMPT,
    BOOK_CHAPTER_SUMMARY_PROMPT,
    BOOK_FOREWARD_PROMPT,
)
from sciphi.writers import RawDataWriter


def save_progress(chapter: str, section: str, subsection: str, output: str):
    """Save progress to a file."""
    with open(f"{output}", "w") as f:
        f.write(f"{chapter}\n{section}\n{subsection or 'None'}")


def load_progress(output_path: str) -> Tuple[str, str, str]:
    """Load progress from a saved file."""
    progress_file = output_path.replace(".md", "_progress.txt")
    if os.path.exists(progress_file):
        with open(progress_file, "r") as f:
            chapter = f.readline().strip()
            section = f.readline().strip()
            subsection = f.readline().strip() or None
        return chapter, section, subsection
    return None, None, None


def get_wiki_related_context(
    query: str, url: str, username: str, password: str, do_wiki: bool
) -> str:
    """Retrieve related context from Wikipedia."""
    return (
        wiki_search_api(url, username, password, query)
        if do_wiki
        else "Related context not available."
    )


def setup_logging(log_level="INFO"):
    """Set up logging."""
    logging.basicConfig(level=log_level.upper())
    logger = logging.getLogger(__name__)
    logger.setLevel(log_level.upper())
    return logger


class TextbookContentGenerator:
    """Generates textbook content from parsed course data."""

    NO_WIKI_TEXT = "Not currently available."

    def __init__(
        self,
        provider="openai",
        model_name="gpt-4-0613",
        data_dir=None,
        toc_dir="output_step_4",
        output_dir="output_step_5",
        max_related_context_to_sample=2_000,
        max_prev_snippet_to_sample=2_000,
        do_wiki=True,
        wiki_server_url=None,
        wiki_username=None,
        wiki_password=None,
        textbook=None,
        log_level="INFO",
        num_threads=None,
    ) -> None:
        self.provider = provider
        self.model_name = model_name
        self.llm_provider = get_default_settings_provider(
            provider=self.provider, model_name=self.model_name
        )

        self.data_dir = data_dir or os.path.join(
            os.path.dirname(__file__), "raw_data"
        )
        self.toc_dir = toc_dir
        self.output_dir = output_dir
        self.textbook = textbook
        self.url = wiki_server_url or os.environ.get("WIKI_SERVER_URL")
        self.username = wiki_username or os.environ.get("WIKI_SERVER_USERNAME")
        self.password = wiki_password or os.environ.get("WIKI_SERVER_PASSWORD")
        self.do_wiki = do_wiki
        self.log_level = log_level.upper()
        self.max_related_context_to_sample = max_related_context_to_sample
        self.max_prev_snippet_to_sample = max_prev_snippet_to_sample

        self.num_threads = num_threads or multiprocessing.cpu_count()

        if self.do_wiki and not all([self.url, self.username, self.password]):
            raise ValueError(
                "Set do_wiki to `False`, or provide Wikipedia server url, username, and password."
            )

        # Constants for sampling
        self.max_related_context_to_sample = max_related_context_to_sample
        self.max_prev_snippet_to_sample = max_prev_snippet_to_sample

        setup_logging(log_level=self.log_level).debug(
            "Finishing initialization"
        )

    @property
    def logger(self) -> logging.Logger:
        return setup_logging(log_level=self.log_level)

    def run(self) -> None:
        """Run the draft book generation process."""
        if self.textbook:
            yml_file_paths = [
                os.path.join(
                    self.data_dir, self.toc_dir, f"{self.textbook}.yaml"
                )
            ]
        else:
            yml_file_paths = glob.glob(
                os.path.join(self.data_dir, self.toc_dir, "**/**/*.yaml")
            )[0:1]

        self.logger.debug(
            f"Running process over a total of {len(yml_file_paths)} files"
        )

        # Use multiprocessing to parallelize the processing of multiple YAML files
        pool = multiprocessing.Pool(processes=self.num_threads)
        pool.map(self.process_yml_file, yml_file_paths)
        pool.close()
        pool.join()

    def process_yml_file(self, yml_file_path: str):
        """Process a single YAML file to generate textbook content."""
        textbook_output_name = self.textbook or os.path.basename(
            yml_file_path
        ).replace(".yaml", "")
        self.logger.debug(f"Processing {yml_file_path}")
        yml_config = load_yaml_file(yml_file_path)

        (
            output_path,
            writer,
            last_chapter,
            last_section,
            last_subsection,
        ) = self.initialize_processing(textbook_output_name)

        traversal_generator = traverse_config(yml_config)
        prev_chapter_config = None
        current_chapter = None
        for counter, elements in enumerate(traversal_generator):
            textbook, chapter, section, subsection = elements
            start_flag = self.update_start_flag(
                chapter,
                section,
                subsection,
                last_chapter,
                last_section,
                last_subsection,
            )

            if not start_flag:
                continue

            self.logger.debug(
                f"Generating step for {chapter} - {section} - {subsection}"
            )

            if counter == 0:
                prev_response, prev_chapter_config = self.handle_foreword(
                    writer, textbook, chapter, prev_chapter_config
                )

            if chapter != current_chapter:
                prev_response = self.handle_chapter_transition(
                    writer,
                    textbook,
                    chapter,
                    current_chapter,
                    prev_chapter_config,
                )
                current_chapter = chapter

            related_context = (
                self.get_related_context(subsection or section)
                if self.do_wiki
                else TextbookContentGenerator.NO_WIKI_TEXT
            )
            step_prompt = self.build_step_prompt(
                textbook,
                chapter,
                section,
                subsection,
                related_context,
                prev_response,
            )

            step_completion = self.persistent_completion(step_prompt)
            prev_response = step_completion
            writer.write(f"{step_completion}\n")

            self.save_current_progress(
                chapter, section, subsection, output_path
            )

        # os.remove(output_path.replace(".md", "_progress.txt"))

    def setup_output_writer(
        self, textbook_name: str
    ) -> Tuple[str, RawDataWriter]:
        """Set up and return the output writer."""
        output_path = os.path.join(
            self.data_dir, self.output_dir, f"{textbook_name}.md"
        )
        if not os.path.exists(os.path.dirname(output_path)):
            os.makedirs(os.path.dirname(output_path))
        self.logger.info(f"Saving textbook to {output_path}")
        writer = RawDataWriter(output_path)
        writer.write("# NOTE - THIS TEXTBOOK WAS AI GENERATED\n")
        writer.write(
            "This textbook was generated using AI techniques. While it aims to be factual and accurate, please verify any critical information. The content may contain errors, biases or harmful content despite best efforts. Please report any issues.\n"
        )
        return output_path, writer

    def handle_foreword(
        self,
        writer: RawDataWriter,
        textbook: str,
        chapter: str,
        prev_chapter_config: dict,
    ) -> Tuple[str, dict]:
        """Handle the foreword section and return the previous response and chapter configuration."""
        self.logger.info(f"Processing {textbook}, Chapter:{chapter}")
        related_context = self.get_related_context(textbook)
        foreward_prompt = BOOK_FOREWARD_PROMPT.format(
            title=textbook,
            related_context=related_context[
                : self.max_related_context_to_sample
            ],
        )
        self.logger.debug("Constructing the foreward...")
        writer.write(f"# {textbook}\n")
        foreward = self._progress_textbook(
            foreward_prompt, writer, "Authored a foreward:\n"
        )
        return foreward, prev_chapter_config

    def handle_chapter_transition(
        self,
        writer: RawDataWriter,
        textbook: str,
        chapter: str,
        current_chapter: str,
        prev_chapter_config: dict,
    ) -> str:
        """Handle chapter transitions and return the previous response."""
        if current_chapter:
            chapter_summary_prompt = BOOK_CHAPTER_SUMMARY_PROMPT.format(
                title=textbook,
                chapter=current_chapter,
                book_context=f"Chapter outline:\n{prev_chapter_config}",
            )
            chapter_completion = self._progress_textbook(
                chapter_summary_prompt,
                writer,
                "Authored a chapter conclusion:\n",
            )
            writer.write(f"{chapter_completion}\n")

        chapter_intro_prompt = BOOK_CHAPTER_INTRODUCTION_PROMPT.format(
            title=textbook,
            chapter=chapter,
            book_context=str(prev_chapter_config),
        )
        chapter_introduction = self.persistent_completion(chapter_intro_prompt)
        self.logger.debug(
            f"Authored a chapter introduction:\n{chapter_introduction}\n\n"
        )
        writer.write(f"{chapter_introduction}\n")
        return chapter_introduction

    def get_related_context(self, query: str) -> str:
        """Retrieve related context from Wikipedia."""
        return get_wiki_related_context(
            query, self.url, self.username, self.password, self.do_wiki
        )

    def build_step_prompt(
        self,
        textbook: str,
        chapter: str,
        section: str,
        subsection: str,
        related_context: str,
        prev_response: str,
    ) -> str:
        """Build and return the step prompt."""
        return BOOK_BULK_PROMPT.format(
            title=textbook,
            chapter=chapter,
            section=section,
            subsection=subsection or "",
            related_context=related_context[
                : self.max_related_context_to_sample
            ],
            book_context=prev_response[: self.max_prev_snippet_to_sample],
        )

    def save_current_progress(
        self, chapter: str, section: str, subsection: str, output_path: str
    ):
        """Save the current progress."""
        save_progress(
            chapter,
            section,
            subsection,
            output_path.replace(".md", "_progress.txt"),
        )

    def persistent_completion(self, prompt: str) -> str:
        return with_retry(lambda: self.llm_provider.get_completion(prompt))

    def initialize_processing(self, textbook_output_name):
        """Initialize processing by setting up the writer and loading progress."""
        output_path, writer = self.setup_output_writer(textbook_output_name)
        last_chapter, last_section, last_subsection = load_progress(
            output_path
        )
        return output_path, writer, last_chapter, last_section, last_subsection

    def update_start_flag(
        self,
        chapter,
        section,
        subsection,
        last_chapter,
        last_section,
        last_subsection,
    ):
        """Update the start flag based on the last processed element."""
        if not last_chapter:
            return True
        return (
            chapter == last_chapter
            and section == last_section
            and (last_subsection == "None" or subsection == last_subsection)
        )

    def _progress_textbook(
        self, prompt: str, writer: RawDataWriter, log_message: str
    ):
        """Progress the textbook generation."""
        result = self.persistent_completion(prompt)
        writer.write(f"{result}\n")
        self.logger.debug(f"{log_message}{result}\n\n")
        return result


if __name__ == "__main__":
    fire.Fire(TextbookContentGenerator)

sciphi/examples/library_of_phi/gen_step_5_draft_book.py
emrgnt-cmplxty/sciphi
# type: ignore
"""
YAML Table of Contents to Textbook Generator

WARNING - This script is very rough and needs significant enhancements to fully productionize
Description:
    This script processes YAML files and generates a detailed Table of Contents for a new textbook accompanying the course.

Usage:
    Command-line interface:
        $ python sciphi/examples/library_of_phi/generate_textbook.py run
Parameters:
    provider (str): 
        The provider to use. Default is 'openai'.
    model (str): 
        The model name to use. Default is 'gpt-3.5-turbo-instruct'.
    parsed_dir (str): 
        Directory containing parsed data. Default is 'raw_data'.
    toc_dir (str): 
        Directory for the table of contents. Default is 'table_of_contents'.
    output_dir (str): 
        Directory for the output. Default is 'output_step_4'.
      (str): 
        Name of the textbook. Default is 'Introduction_to_Deep_Learning'.
    max_related_context_to_sample (int): 
        Maximum context to sample. Default is 2000.
    max_prev_snippet_to_sample (int): 
        Maximum previous snippet to sample. Default is 2000.
    do_wiki (bool): 
        Whether to use Wikipedia. Default is True.
    wiki_server_url (str): 
        Wikipedia server URL. Uses environment variable "WIKI_SERVER_URL" if not provided.
    wiki_username (str): 
        Wikipedia username. Uses environment variable "WIKI_SERVER_USERNAME" if not provided.
    wiki_password (str): 
        Wikipedia password. Uses environment variable "WIKI_SERVER_PASSWORD" if not provided.
    log_level (str): 
        Logging level. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL. Default is 'INFO'.
"""
import os

import fire

from sciphi.core.utils import get_data_dir
from sciphi.examples.library_of_phi.gen_step_5_draft_book import (
    TextbookContentGenerator,
)


class TextbookContentGeneratorSimplified:
    def __init__(self) -> None:
        pass

    def run(
        self,
        provider="openai",
        model_name="gpt-4-0613",
        toc_dir="table_of_contents",
        textbook="Aerodynamics_of_Viscous_Fluids",
        max_related_context_to_sample=2_000,
        max_prev_snippet_to_sample=2_000,
        do_wiki=True,
        wiki_server_url=None,
        wiki_username=None,
        wiki_password=None,
        log_level="INFO",
    ):
        """Runs the generation process to create a textbook from the YAML table of contents."""
        TextbookContentGenerator(
            provider,
            model_name,
            data_dir=os.path.join(get_data_dir(), "library_of_phi"),
            toc_dir=toc_dir,
            output_dir="",
            textbook=textbook,
            max_related_context_to_sample=max_related_context_to_sample,
            max_prev_snippet_to_sample=max_prev_snippet_to_sample,
            do_wiki=do_wiki,
            wiki_server_url=wiki_server_url,
            wiki_username=wiki_username,
            wiki_password=wiki_password,
            log_level=log_level,
        ).run()


if __name__ == "__main__":
    fire.Fire(TextbookContentGeneratorSimplified)

sciphi/examples/library_of_phi/generate_textbook.py
emrgnt-cmplxty/sciphi
# type: ignore
import textwrap

SYLLABI_CREATION_PROMPT = textwrap.dedent(
    """
Take a deep breath, then closely examine the configuration that follows. You will be asked to produce a similar configuration as part of this task, so pay close attention to the format.

```yaml
course:
    Quantum Physics I:
        topics:
            - Basic Features of Quantum Mechanics:
                subtopics:
                    - Linearity
                    - Complex Numbers
                    - Non-deterministic
                    - Superposition
                    - Entanglement
            - Experimental Basis of Quantum Physics:
                subtopics:
                    - Two-slit Experiments
                    - Mach-Zehnder Interferometer
                    - Elitzur-Vaidman Bombs
                    - Photoelectric Effect
                    - Compton Scattering
                    - de Broglie Wavelength
            - Wave Mechanics:
                subtopics:
                    - Galilean Transformation of de Broglie Wavelength
                    - Wave-packets and Group Velocity
                    - Matter Wave for a Particle
                    - Momentum and Position Operators
                    - Schrödinger Equation
            - Interpretation of the Wavefunction:
                subtopics:
                    - Probability Density
                    - Probability Current
                    - Current Conservation
                    - Hermitian Operators
            - Expectation Values and Uncertainty:
                subtopics:
                    - Expectation Values of Operators
                    - Time Evolution of Wave-packets
                    - Fourier Transforms
                    - Parseval Theorem
                    - Uncertainty Relation
            - Quantum Physics in One-dimensional Potentials:
                subtopics:
                    - Stationary States
                    - Boundary Conditions
                    - Particle on a Circle
                    - Infinite Square Well
                    - Finite Square Well
                    - Semiclassical Approximations
                    - Numerical Solution by the Shooting Method
                    - Delta Function Potential
                    - Simple Harmonic Oscillator
                    - Reflection and Transmission Coefficients
                    - Ramsauer Townsend Effect
                    - 1D Scattering and Phase Shifts
                    - Levinson’s Theorem
            - Angular Momentum and Central Potentials:
                subtopics:
                    - Resonances and Breit-Wigner Distribution
                    - Central Potentials
                    - Algebra of Angular Momentum
                    - Legendre Polynomials
                    - Hydrogen Atom
                    - Energy Levels Diagram
                    - Virial Theorem
                    - Circular Orbits and Eccentricity
                    - Discovery of Spin
```

Given the following context, deduce the new configuration entry for the course "{course_name}". Be sure to fill in the appropriate topics and subtopics.

```md
{context}
```

Notes:
- Filter out any erroneous content like "QUIZ" or "TEST" that might appear in the syllabus. 
- Attempt to match the length and specificity of the above example; add your own context if necessary to accomplish this or remove extra context.
- You should target **5-10 main topics with 6-10 subtopics each**.

### Response:
"""
)


TABLE_OF_CONTENTS_DRAFT_PROMPT = """
### Instructions:
Below the course syllabus for {course_name} is shown as a yaml snippet.

```yaml
{context}
```

Pretend you are a prolific author, your task is to translate the shown syllabus into a very detailed Table of Contents for a new textbook that is meant to accompany the course.
Be sure to pick a fitting name for the textbook. Add additional topics so that the table of contents provides much more information than the original syllabus.
Return your final result in an easily parseable yaml format, your final yaml must match the above with course -> textbook, and topics -> chapters, subtopics -> sections, and an additional new key: subsubtopics -> subsections.

### Response:
"""


BOOK_FOREWARD_PROMPT = """
### Instructions:
You are a writing a book titled "{title}". You are currently writing the foreward for the book:

#  Title: {title}


To assist you in this task, you have been provided the context bellow:

### Related Context
```
{related_context}
```

Notes:
- The book is being written in the popular Markdown format.
- The context may be truncated and is meant only to provide a starting point. Feel free to expand on it or take the response in any direction that fits the prompt, but keep it in a voice that is appropriate for an advanced undergraduate course at MIT.
- Avoid making any factual claims or opinions without proper citations or context to support them, stick to the proposed context.
- Begin your response with `## Foreward`

### Response:
"""


TABLE_OF_CONTENTS_CLEAN_PROMPT = """
### Instructions:
Below is a table of contents for the textbook "Aerodynamics of Viscous Fluids: A Comprehensive Study". This table of contents has a specified format consisting of a "textbook" key, followed by the textbook title as a key. Nested under the textbook title are the keys for chapters, sections, and subsections. The table is meant to follow a yaml format, but spacing has been removed to save space.

textbook: 
"Aerodynamics of Viscous Fluids: A Comprehensive Study":

chapters:
- Chapter 1: Fundamental Theorem of Kinematics:
sections:
- Section: 1.1 Convection:
subsections:
- 1.1a Introduction to Convection
- 1.1b Boundary Layer Convection
- 1.1c Applications of Convection

- Section: 1.2 Vorticity:
subsections:
- 1.2a Definition of Vorticity
- 1.2b Vorticity Transport Equation
- 1.2c Vorticity Generation

- Section: 1.3 Strain:
subsections:
- 1.3a Introduction to Strain
- 1.3b Strain Rate Tensor
- 1.3c Rate of Deformation Tensor

- Chapter 2: Eulerian vs. Lagrangian Description:
sections:
- Section: 2.1 Convection Relations:
subsections:
- 2.1a Eulerian and Lagrangian Descriptions
- 2.1b Material Derivative
- 2.1c Applications and Differences

- Section: 2.2 Frame of Reference:
subsections:
- 2.2a Fixed Frame
- 2.2b Moving Frame
- 2.2c Advantages and Limitations

- Section: 2.3 Mathematical Representations:
subsections:
- 2.3a Transformation Equations
- 2.3b Jacobian Matrices
- 2.3c Use in Kinematics

- Chapter 3: Conservation Laws:
sections:
- Section: 3.1 Conservation of Mass:
subsections:
- 3.1a Continuity Equation
- 3.1b Incompressible Flow
- 3.1c Compressible Flow
- Section: 3.2 Conservation of Momentum:
subsections:
- 3.2a Navier-Stokes Equations
- 3.2b Reynolds Transport Theorem
- 3.2c Pressure and Viscous Forces
- Section: 3.3 Stress Tensor:
subsections:
- 3.3a Definition of Stress Tensor
- 3.3b Viscous and Pressure Stresses
- 3.3c Reynolds Stress Tensor

- Chapter 4: Viscosity and Newtonian Fluids:
sections:
- Section: 4.1 Vorticity and Circulation:
subsections:
- 4.1a Vorticity and Circulation Equations
- 4.1b Kelvin's Circulation Theorem
- 4.1c Vorticity Distribution
- 4.1d Circulation and Vorticity Relations
- Section: 4.2 Newtonian vs. Non-Newtonian Fluids:
subsections:
- 4.2a Definition and Characteristics
- 4.2b Models of Non-Newtonian Fluids
- 4.2c Applications and Differences
- Section: 4.3 Shear and Bulk Viscosity:
subsections:
- 4.3a Definitions and Differences
- 4.3b Measurement Techniques
- 4.3c Importance in Fluid Mechanics

- Chapter 5: Navier-Stokes Equations:
sections:
- Section: 5.1 Physical Parameters:
subsections:
- 5.1a Dimensionless Parameters
- 5.1b Reynolds Number
- 5.1c Mach Number
- Section: 5.2 Dynamic Similarity:
subsections:
- 5.2a Scaling Laws
- 5.2b Model Testing
- 5.2c Similarity Criteria

- Chapter 6: Dimensional Analysis:
sections:
- Section: 6.1 Dominant Balance:
subsections:
- 6.1a Balance of Forces
- 6.1b Balance of Stresses
- Section: 6.2 Viscous Flow Classification:
subsections:
- 6.2a Laminar Flow
- 6.2b Turbulent Flow
- Section: 6.3 Buckingham's Pi Theorem:
subsections:
- 6.3a Introduction and Theory
- 6.3b Application in Fluid Mechanics
- 6.3c Examples and Case Studies

- Chapter 7: Thin Shear Layer Equations:
sections:
- Section: 7.1 TSL Coordinates:
subsections:
- 7.1a Definition of TSL Coordinates
- 7.1b TSL Navier-Stokes Equations
- Section: 7.2 Boundary Conditions:
subsections:
- 7.2a No-Slip Condition
- 7.2b Impermeability Condition
- 7.2c Kutta Condition
- Section: 7.3 Shear Layer Categories:
subsections:
- 7.3a Blasius Shear Layer
- 7.3b Falkner-Skan Shear Layer
- 7.3c Displacement Shear Layer

- Chapter 8: Local Scaling:
sections:
- Section: 8.1 Falkner-Skan Flows:
subsections:
- 8.1a Introduction to Falkner-Skan Flows
- 8.1b Boundary Layer Equations
- 8.1c Solution Techniques

- Section: 8.2 Scaling Laws in Fluid Mechanics:
subsections:
- 8.2a Basic Principles of Scaling
- 8.2b Applications in Different Flow Regimes
- 8.2c Limitations and Challenges

- Section: 8.3 Similarity Solutions:
subsections:
- 8.3a Introduction to Similarity Solutions
- 8.3b Methods to Achieve Similarity
- 8.3c Examples in Fluid Dynamics

- Chapter 9: ODE's, PDE's, and Boundary Conditions:
sections:
- Section: 9.1 Well-Posedness:
subsections:
- 9.1a Definition of Well-Posedness
- 9.1b Initial and Boundary Conditions
- Section: 9.2 Partial Differential Equations:
subsections:
- 9.2a Classification of PDEs
- 9.2b Elliptic, Parabolic, and Hyperbolic Equations
- Section: 9.3 Boundary Conditions:
subsections:
- 9.3a Dirichlet Boundary Condition
- 9.3b Neumann Boundary Condition
- 9.3c Mixed Boundary Condition

- Chapter 10: Numerical Methods for ODE's:
sections:
- Section: 10.1 Discretization:
subsections:
- 10.1a Finite Difference Method
- 10.1b Finite Element Method
- 10.1c Finite Volume Method
- Section: 10.2 Stability:
subsections:
- 10.2a Von Neumann Stability Analysis
- 10.2b CFL Condition
- 10.2c Implicit vs. Explicit Methods

- Chapter 11: Integral Methods:
sections:
- Section: 11.1 Integral Momentum Equation:
subsections:
- 11.1a Integral Boundary Layer Equations
- 11.1b Blasius Solution
- 11.1c Falkner-Skan Solution
- Section: 11.2 Thwaites' Method:
subsections:
- 11.2a Introduction to Thwaites' Method
- 11.2b Boundary Layer Growth
- 11.2c Thwaites' Laminar Boundary Layer Solution

- Chapter 12: Dissipation Methods:
sections:
- Section: 12.1 Integral Kinetic Energy Equation:
subsections:
- 12.1a Turbulent Dissipation
- 12.1b Shear Stress Distribution
- 12.1c Integral Energy Equation

- Section: 12.2 Turbulent Dissipation Mechanisms:
subsections:
- 12.2a Basics of Turbulence and Eddies
- 12.2b Cascade Mechanism
- 12.2c Effects on Boundary Layer

- Section: 12.3 Experimental and Numerical Techniques:
subsections:
- 12.3a Techniques for Measuring Dissipation
- 12.3b Numerical Methods and Challenges
- 12.3c Comparison of Experimental and Numerical Results

- Chapter 13: Asymptotic Perturbation Theory:
sections:
- Section: 13.1 Higher-Order Effects:
subsections:
- 13.1a Introduction to Perturbation Theory
- 13.1b Asymptotic Expansion
- 13.1c Higher-Order Corrections
- Section: 13.2 Boundary Layer Perturbation:
subsections:
- 13.2a Introduction to Boundary Layer Perturbation
- 13.2b Application in Viscous Flows
- 13.2c Challenges and Limitations
- Section: 13.3 Nonlinear Perturbations:
subsections:
- 13.3a Introduction to Nonlinear Perturbations
- 13.3b Application in Fluid Dynamics
- 13.3c Advanced Methods and Techniques

- Chapter 14: 2D Interaction Models:
sections:
- Section: 14.1 Displacement Body:
subsections:
- 14.1a Flow Past a Cylinder
- 14.1b Flow Past an Airfoil
- Section: 14.2 Transpiration:
subsections:
- 14.2a Transpiration Boundary Condition
- 14.2b Transpiration Effects on Flow Field
- Section: 14.3 Form Drag:
subsections:
- 14.3a Definition of Form Drag
- 14.3b Pressure Drag
- 14.3c Friction Drag
- Section: 14.4 Stall Mechanisms:
subsections:
- 14.4a Boundary Layer Separation
- 14.4b Flow Reattachment
- 14.4c Dynamic Stall

- Chapter 15: IBLT Solution Techniques:
sections:
- Section: 15.1 Iteration Stability:
subsections:
- 15.1a Stability Analysis
- 15.1b Convergence Criteria
- 15.1c Solution Techniques
- Section: 15.2 Fully-Coupled Iteration:
subsections:
- 15.2a Introduction to Fully-Coupled Iteration
- 15.2b Fluid-Structure Interaction
- 15.2c Aeroelasticity
- Section: 15.3 3-D IBLT:
subsections:
- 15.3a Three-Dimensional Boundary Layers
- 15.3b Transition Mechanisms in 3D Flows

- Chapter 16: Small-Perturbation Theory:
sections:
- Section: 16.1 Orr-Sommerfeld Equation:
subsections:
- 16.1a Definition of Orr-Sommerfeld Equation
- 16.1b Linear Stability Analysis
- 16.1c Growth Rates and Frequencies
- Section: 16.2 Linear Instabilities:
subsections:
- 16.2a Introduction to Linear Instabilities
- 16.2b Applications in Viscous Flows
- 16.2c Detection and Control Methods
- Section: 16.3 Perturbation in Compressible Flows:
subsections:
- 16.3a Basics of Compressible Flow Perturbations
- 16.3b Challenges in High-Speed Flows
- 16.3c Recent Advances and Developments

- Chapter 17: Boundary Conditions, Homogeneity, Solution Techniques:
sections:
- Section: 17.1 Transition Mechanisms:
subsections:
- 17.1a Laminar-Turbulent Transition
- 17.1b Transition Prediction Methods
- Section: 17.2 Transition Prediction:
subsections:
- 17.2a Stability Analysis
- 17.2b Experimental Methods
- 17.2c Computational Methods

- Chapter 18: Reynolds Averaging:
sections:
- Section: 18.1 Prandtl's Analogy:
subsections:
- 18.1a Reynolds-Averaged Navier-Stokes Equations
- 18.1b Turbulent Stresses
- 18.1c Reynolds Stress Modeling

- Chapter 19: Turbulent BL Structure:
sections:
- Section: 19.1 Wake:
subsections:
- 19.1a Wake Formation
- 19.1b Wake Characteristics
- Section: 19.2 Wall Layers:
subsections:
- 19.2a Viscous Sublayer
- 19.2b Buffer Layer
- 19.2c Overlap Layer
- Section: 19.3 Inner, Outer Variables:
subsections:
- 19.3a Velocity and Length Scales
- 19.3b Wall Coordinates
- 19.3c Outer Variables
- Section: 19.4 Effects of Roughness:
subsections:
- 19.4a Roughness Effects on Boundary Layer
- 19.4b Skin Friction and Heat Transfer

- Chapter 20: Equilibrium BL's:
sections:
- Section: 20.1 Clauser Hypothesis:
subsections:
- 20.1a Equilibrium Boundary Layer Concept
- 20.1b Clauser's Equilibrium Assumptions
- Section: 20.2 Dissipation Formulas:
subsections:
- 20.2a Turbulent Energy Dissipation
- 20.2b Dissipation in Equilibrium Boundary Layers
- Section: 20.3 Integral Closure:
subsections:
- 20.3a Reynolds Stress Closure
- 20.3b Eddy Viscosity Models
- 20.3c Mixing Length Models
...

Your task is to inspect the table of contents that is provided below and to produce a new table of contents which is formatted EXACTLY as the example above. 

{table_of_contents}

Further, if the table of contents is less than 20 chapters, or lacking sufficient detail in any given chapter, then your task is to coherently expand the table and re-arrange it so that it is 20+ chapters, with 3-4 sections and 3-4 subsections per chapter, formatted exactly as the example above.

If the table of contents is already in the correct format and complete, reproduce it as-is with the new formatting.


Notes:
- Chapters should be numbered `Chapter 1`, `Chapter 2`, etc.
- Sections should be numbered `Section 1.1`, `Section 1.2`, etc.
- Subsections should be numbered `1.1a`, `1.1b`, etc.
- The table of contents should contain no indentations.
- The table of contents should be 20 chapters or more, and have 3-4 sections and 3-4 subsections per chapter.

### Response:
"""

BOOK_BULK_PROMPT = """
### Instructions:
You are a writing a book titled "{title}". You are currently writing the chapter and section shown below:

#  Title: {title}

## Chapter: {chapter}

### Section: {section}

### Subsection (optional): {subsection}

To assist you in writing the chapter, you have been provided with some related context and recent chapter contents bellow:

### Related Context
```
{related_context}
```

### Last textbook section content:
```
{book_context}
```

Notes:
- The book is being written in the popular Markdown format.
- The context may be truncated and is meant only to provide a starting point. Feel free to expand on it or take the response in any direction that fits the prompt, but keep it in a voice that is appropriate for an advanced undergraduate course at MIT.
- Avoid making any factual claims or opinions without proper citations or context to support them, stick to the proposed context.
- Format ALL math equations with the $ and $$ delimiters to insert math expressions in TeX and LaTeX style syntax. This content is then rendered using the highly popular MathJax library. E.g. write inline math like `$y_j(n)$` and equations like `$$
\\Delta w = ...
$$
- If starting a new section, include `### [Section Title]`
- If starting a new subsection, include `#### [Subsection Title]`
`

### Response:
"""

# Prompt for the LLM
BOOK_CHAPTER_SUMMARY_PROMPT = """
### Instructions:
You are a writing a book titled "{title}". You are tasked with writing a several paragraph CONCLUSION FOR THE CHAPTER shown below:

#  Title: {title}

## Chapter: {chapter}

To assist you in this task, you have been provided the context bellow:

### Previously written chapter:
```
{book_context}
```

Following your authoring of the conclusion, write 5 exercises which align with the context of the chapter. Format each with a header `#### Exercise 1` etc.

Notes:
- The book is being written in the popular Markdown format.
- Avoid making any factual claims or opinions without proper citations or context to support them, stick to the proposed context.
- Format ALL math equations with the $ and $$ delimiters to insert math expressions in TeX and LaTeX style syntax. This content is then rendered using the highly popular MathJax library. E.g. write inline math like `$y_j(n)$` and equations like `$$
\\Delta w = ...
$$
- Start the conclusion the Chapter with a header that reads `### Conclusion`, start the exercises with a header that reads `### Exercises`.

`

### Response:
"""

# Prompt for the LLM
BOOK_CHAPTER_INTRODUCTION_PROMPT = """
### Instructions:
You are a writing a book titled "{title}". You are currently writing a several paragraph introducton for the chapter shown below (avoid going into too much detail):

#  Title: {title}

## Chapter: {chapter}

To assist you in this task, you have been provided the context which will be covered in this chapter:

### Chapter section topics covered:
```
{book_context}
```

Notes:
- The book is being written in the popular Markdown format.
- Avoid making any factual claims or opinions without proper citations or context to support them, stick to the proposed context.
- Format ALL math equations with the $ and $$ delimiters to insert math expressions in TeX and LaTeX style syntax. This content is then rendered using the highly popular MathJax library. E.g. write inline math like `$y_j(n)$` and equations like `$$
\\Delta w = ...
$$
- Include the chapter title at the top of your output, formatted as  `## Chapter: [Title]`,  `### Introduction` below that.
`

### Response:
"""

sciphi/examples/library_of_phi/prompts.py
emrgnt-cmplxty/sciphi
"""
MIT OCW Course Data Scraper

Description:
    This script is designed to scrape course data from the MIT OpenCourseWare (OCW) website.
    It extracts details like course number, title, URL, resource level, page contents, and more.

Usage:
    Command-line interface:
        $ poetry run python sciphi/examples/library_of_phi/raw_data/ocw_scraper.py scrape \
            --input_file_name=ocw_courses.html \
            --output_file_name=scraped_ocw.jsonl \
            --sleep_time=1 \
            --log_level=INFO

Parameters:
    data_directory (Optional[str], default=None): 
        Path to the directory where the input file is located 
        and where the output file will be saved. 
        If not specified, defaults to the directory of this script.

    input_file_name (str, default="ocw_courses.html"): 
        Name of the input HTML file containing MIT OCW course listings.
    
    output_file_name (str, default="ocw_scraped.jsonl"): 
        Name of the output JSONL file where scraped data will be saved.
        
    sleep_time (int, default=0): 
        Time (in seconds) to pause between scraping consecutive courses.
    
    log_level (str, default="INFO"): 
        Logging level for the scraper. Can be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL.
"""
import logging
import os
import re
import time
from typing import Dict, List, Optional, Union

import fire
import requests
from bs4 import BeautifulSoup

from sciphi.writers import JsonlDataWriter

# Constants
BASE_URL = "https://ocw.mit.edu"
USER_AGENT = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
)
HEADERS = {"User-Agent": USER_AGENT}
PAGE_TYPES = [
    "readings",
    "lecture-notes",
    "recitations",
    "assignments",
    "exams",
    "syllabus",
    "calendar",
    "projects",
]

logging.basicConfig(level=logging.INFO)


class Scraper:
    """Scraper for MIT OCW course data."""

    def __init__(self, sleep_time: int = 0, log_level: str = "INFO"):
        self.sleep_time = sleep_time
        logging.basicConfig(level=getattr(logging, log_level.upper()))

    def scrape(
        self,
        data_directory: Optional[str] = None,
        input_file_name: str = "ocw_courses.html",
        output_file_name: str = "ocw_scraped.jsonl",
    ):
        """Scrape course data from MIT OCW and save to an output file."""

        if not data_directory:
            data_directory = os.path.dirname(os.path.abspath(__file__))

        with open(os.path.join(data_directory, input_file_name)) as f_open:
            html_content = f_open.read()

        soup = BeautifulSoup(html_content, "html.parser")
        articles = soup.find_all("article")

        writer = JsonlDataWriter(
            os.path.join(data_directory, output_file_name)
        )

        for article in articles:
            course_data = Scraper.extract_course_data(article)
            writer.write([course_data])
            time.sleep(self.sleep_time)

    @staticmethod
    def get_soup(page_url: str) -> Optional[BeautifulSoup]:
        """Retrieve and parse a webpage."""
        try:
            response = requests.get(page_url, headers=HEADERS)
            response.raise_for_status()
            return BeautifulSoup(response.text, "html.parser")
        except requests.RequestException as e:
            logging.error(f"Failed to retrieve {page_url}. Error: {e}")
            return None

    @staticmethod
    def read_page(
        page_url: str,
    ) -> Dict[str, Union[str, List[Dict[str, str]]]]:
        """Read the given page URL and return a dictionary of the extracted data."""
        logging.info(f"Scraping {page_url}")
        soup = Scraper.get_soup(page_url)
        if not soup:
            return {"leading_text": "", "table_rows": []}

        result: dict = {
            "leading_text": Scraper._extract_leading_text(soup),
            "table_rows": Scraper._extract_table_rows(soup),
        }

        if page_url.endswith("syllabus/"):
            logging.debug("Scraping the syllabus")
            try:
                result["topics"] = Scraper._get_topics_and_depths(soup)
            except Exception as e:
                logging.warning(f"Failed to extract topics. Error: {e}")
                result["topics"] = {}

        return result

    @staticmethod
    def _extract_text(
        element: BeautifulSoup, tag: str, class_: str
    ) -> Optional[str]:
        """Extract text from an element with a specific tag and class."""
        found_element = element.find(tag, class_=class_)
        return found_element.get_text(strip=True) if found_element else None

    def _extract_leading_text(soup: BeautifulSoup) -> str:
        """Extract leading text from the soup."""
        main_section = soup.find("main", id="course-content-section")
        if main_section:
            paragraphs = main_section.find_all("p")
            leading_text = " ".join(
                paragraph.get_text(strip=True) for paragraph in paragraphs
            )
            return Scraper._clean_text(leading_text)
        return ""

    def extract_course_data(
        article: BeautifulSoup,
    ) -> Dict[str, Union[str, List[Dict[str, str]]]]:
        """Extract course data from an article tag."""
        logging.debug("Extracting course data...")

        course_num = Scraper._extract_text(article, "div", "col-2 course-num")
        course_title_div = article.find("div", class_="course-title")
        course_url = (
            course_title_div.find("a")["href"] if course_title_div else None
        )
        course_title = Scraper._extract_text(article, "div", "course-title")
        resource_level = Scraper._extract_text(
            article, "div", "col-2 resource-level"
        )

        course_data = {
            "course_number": course_num,
            "course_title": course_title,
            "course_url": course_url,
            "resource_level": resource_level,
            "page_contents": {},
        }

        ocw_link = (
            f"{BASE_URL}{course_url.rstrip('/')}" if course_url else None
        )
        for page_type in PAGE_TYPES:
            page_url = f"{ocw_link}/pages/{page_type}/"
            try:
                course_data["page_contents"][page_type] = Scraper.read_page(
                    page_url
                )
            except requests.RequestException:
                pass

        return course_data

    @staticmethod
    def _extract_table_rows(soup: BeautifulSoup) -> List[Dict[str, str]]:
        """Extract table rows from the soup."""
        logging.debug("Extracting table rows...")

        rows = soup.find_all("tr")
        table_data = []

        for row in rows:
            try:
                note_number = Scraper._clean_text(
                    row.find_all("td")[0].get_text(strip=True)
                )
                topic_col = row.find_all("td")[1]
                topic = Scraper._clean_text(
                    topic_col.get_text(strip=True).split("(")[0].strip()
                )
                link = topic_col.find("a")
                note_url = link["href"] if link else None
                table_data.append(
                    {
                        "note_number": note_number,
                        "topic": topic,
                        "note_url": note_url,
                    }
                )
            except IndexError:
                logging.debug(
                    "Skipping row that doesn't match the expected format"
                )
                pass  # Skip rows that don't match the expected format

        return table_data

    @staticmethod
    def _clean_text(text: str) -> str:
        """Clean the given text."""
        text = text.replace("\xa0", " ")
        return re.sub(r"\s+", " ", text).strip()

    @staticmethod
    def _get_topics_and_depths(soup: BeautifulSoup) -> Dict[str, int]:
        """Extract topics and their depths from the soup."""
        topics = soup.select("span.topic-text-wrapper > a.course-info-topic")
        return {
            topic.get_text(strip=True): len(
                topic.find_parent("li").find_parents("ul")
            )
            - 1
            for topic in topics
        }


def main():
    """Main function."""
    fire.Fire(Scraper)


if __name__ == "__main__":
    main()

sciphi/examples/library_of_phi/raw_data/ocw_scraper.py
emrgnt-cmplxty/sciphi
"""A module for populating ChromaDB with a dataset."""
import os
from concurrent.futures import ThreadPoolExecutor
from threading import current_thread
from typing import Generator

import chromadb
import dotenv
import openai
from chromadb.config import Settings
from datasets import Dataset, load_dataset
from openai.embeddings_utils import get_embeddings
from retrying import retry

from sciphi.core.utils import get_configured_logger

dotenv.load_dotenv()


def chunk_text(text: str, chunk_size: int) -> list[str]:
    """Chunk a text into a list of strings of size chunk_size."""
    return [text[i : i + chunk_size] for i in range(0, len(text), chunk_size)]


# Define a retrying decorator with specific parameters
@retry(
    stop_max_attempt_number=3, wait_fixed=2000
)  # Retries 3 times with a 2-second wait between retries
def robust_get_embeddings(chunks: list[str], engine: str) -> list[list[float]]:
    try:
        return get_embeddings(chunks, engine=engine)
    except Exception as e:
        logger.error(
            f"Failed to get embeddings for chunks {chunks}, with error {e}"
        )
        raise  # Reraise the exception to be caught by the retrying mechanism


def worker(worker_args: tuple) -> None:
    """Worker function to populate ChromaDB with a batch of entries."""
    thread_name = current_thread().name
    (
        collection,
        entries_batch,
        chunk_size,
        batch_size,
        max_embedding_batch_size,
        embedding_engine,
        logger,
        logger_interval,
    ) = worker_args

    local_buffer: dict[str, list] = {
        "documents": [],
        "embeddings": [],
        "metadatas": [],
        "ids": [],
    }

    logger.info(
        f"Iterating over a batch of size {len(entries_batch)} in {thread_name}..."
    )

    for n_samples_iter_local, (entry, raw_ids) in enumerate(
        entries_batch, start=1
    ):
        chunks = chunk_text(entry["code"], chunk_size)
        if n_samples_iter_local % logger_interval == 0:
            logger.info(
                f"Thread {thread_name} processed {n_samples_iter_local} samples"
            )

        local_buffer["documents"].extend(chunks)

        for i in range(0, len(chunks), max_embedding_batch_size):
            batch_of_chunks = chunks[i : i + max_embedding_batch_size]
            local_buffer["embeddings"].extend(
                get_embeddings(batch_of_chunks, engine=embedding_engine)
            )

        local_buffer["metadatas"].extend(
            [
                {
                    "package": entry["package"],
                    "path": entry["path"],
                    "filename": entry["filename"],
                }
            ]
            * len(chunks)
        )
        local_buffer["ids"].extend(raw_ids)

        # Write to database in chunks
        if len(local_buffer["documents"]) >= batch_size:
            logger.debug(f"Inserting ids = {local_buffer['ids']}")
            try:
                collection.add(
                    embeddings=local_buffer["embeddings"],
                    documents=local_buffer["documents"],
                    metadatas=local_buffer["metadatas"],
                    ids=local_buffer["ids"],
                )
            except Exception as e:
                logger.error(
                    f"Failed to insert ids = {local_buffer['ids']}, with {e} skipping."
                )
            local_buffer = {
                "documents": [],
                "embeddings": [],
                "metadatas": [],
                "ids": [],
            }


def batch_dataset(
    dataset: Dataset, batch_size: int, parsed_ids: set[str]
) -> Generator[list[tuple[dict, list[str]]], None, None]:
    # Count the number of chunks we have already parsed
    n_entries = len(parsed_ids)
    if n_entries > 0:
        logger.info(f"Loaded {n_entries} entries from ChromaDB")

    batch = []
    n_entries = 0
    for entry in dataset:
        chunks = chunk_text(entry["code"], chunk_size)
        raw_ids = [
            f"id_{i}" for i in range(n_entries, n_entries + len(chunks))
        ]
        n_entries += len(chunks)

        if set(raw_ids).issubset(parsed_ids):
            logger.debug(f"Skipping ids = {raw_ids} as they already exist")
            continue
        batch.append((entry, raw_ids))
        if len(batch) == batch_size:
            yield batch
            batch = []
    if batch:
        yield batch


if __name__ == "__main__":
    # TODO - Move to proper CLI based approach, like Fire.
    # For now, we are getting it running quick and dirty.

    # Chroma environment variables
    chroma_addr = os.environ["CHROMA_REMOTE_ADDR"]
    chroma_port = os.environ["CHROMA_REMOTE_PORT"]
    chroma_token = os.environ["CHROMA_TOKEN"]
    chroma_auth_provider = os.environ["CHROMA_AUTH_PROVIDER"]

    # OpenAI environment variables
    openai_api_key = os.environ["OPENAI_API_KEY"]
    openai.api_key = openai_api_key
    embedding_engine = "text-embedding-ada-002"

    # HF dataset
    dataset_name = "vikp/pypi_clean"
    streaming = False
    # Script variables
    # For chunking the code into smaller pieces
    chunk_size = 2048
    # For batching the embedding calls & inserts into ChromaDB
    batch_size = 64
    batches_per_split = 8
    max_embedding_batch_size = 2048
    # Process dataset in multiple threads
    num_threads = 6
    # For logging
    # TODO - Modify to sure we are logging by-process
    log_level = "INFO"
    sample_log_interval = 100

    # Output collectionn name
    collection_name = (
        f"{dataset_name.replace('/', '_')}_chunk_size_eq_{chunk_size}"
    )

    logger = get_configured_logger("populate_chroma_db", log_level)
    logger.info("Starting to populate ChromaDB")

    if not chroma_token or not chroma_addr or not chroma_port:
        raise ValueError(
            f"ChromaDB environment variables not set correctly, found: chroma_token={chroma_token}, chroma_addr={chroma_addr}, chroma_port={chroma_port}"
        )

    if not openai_api_key:
        raise ValueError(
            "OpenAI API key not found. Please set the OPENAI_API_KEY environment variable."
        )

    client = chromadb.HttpClient(
        host=chroma_addr,
        port=chroma_port,
        settings=Settings(
            chroma_client_auth_provider=chroma_auth_provider,
            chroma_client_auth_credentials=chroma_token,
        ),
    )

    try:
        collection = client.create_collection(name=collection_name)
    except Exception as e:
        logger.info(
            f"Collection {collection_name} likely already exists, skipping creation. For completeness, here is the exception: {e}"
        )
        collection = client.get_collection(name=collection_name)
    parsed_ids = set(collection.get(include=[])["ids"])

    dataset = load_dataset(dataset_name, streaming=streaming)
    if not streaming:
        dataset = dataset["train"].shuffle(seed=42)

    logger.info("Creating the dataset batches...")

    with ThreadPoolExecutor(max_workers=num_threads) as executor:
        split_size = batches_per_split * batch_size
        args_for_workers = (
            (
                collection,
                batch,
                chunk_size,
                batch_size,
                max_embedding_batch_size,
                embedding_engine,
                logger,
                sample_log_interval,
            )
            for batch in batch_dataset(dataset, split_size, parsed_ids)
        )
        # The map method blocks until all results are returned
        list(executor.map(worker, args_for_workers))

sciphi/examples/populate_chroma/runner.py
emrgnt-cmplxty/sciphi
from sciphi.interface.anthropic_interface import AnthropicLLMInterface
from sciphi.interface.base import LLMInterface, ProviderConfig, ProviderName
from sciphi.interface.hugging_face_interface import HuggingFaceLLMInterface
from sciphi.interface.interface_manager import InterfaceManager
from sciphi.interface.litellm_interface import LiteLLMInterface
from sciphi.interface.llama_index_interface import LlamaIndexInterface
from sciphi.interface.openai_interface import OpenAILLMInterface
from sciphi.interface.vllm_interface import vLLMInterface

__all__ = [
    "InterfaceManager",
    "ProviderName",
    "ProviderConfig",
    "LLMInterface",
    # Concrete Providers
    "AnthropicLLMInterface",
    "HuggingFaceLLMInterface",
    "LlamaIndexInterface",
    "OpenAILLMInterface",
    "vLLMInterface",
    "LiteLLMInterface",
]

sciphi/interface/__init__.py
emrgnt-cmplxty/sciphi
"""A module for interfacing with the Anthropic API"""
from sciphi.interface.base import LLMInterface, ProviderName
from sciphi.interface.interface_manager import llm_provider
from sciphi.llm import AnthropicConfig, AnthropicLLM, ModelName


@llm_provider
class AnthropicLLMInterface(LLMInterface):
    """A class to interface with the Anthropic API."""

    provider_name = ProviderName.ANTHROPIC
    supported_models = [
        ModelName.CLAUDE_INSTANT_1,
        ModelName.CLAUDE_2,
    ]

    def __init__(
        self,
        config: AnthropicConfig = AnthropicConfig(),
    ) -> None:
        super()
        self._model = AnthropicLLM(
            config,
        )

    def get_completion(self, prompt: str) -> str:
        """Get a completion from the remote Anthropic provider."""
        return self.model.get_instruct_completion(prompt)

    @property
    def model(self) -> AnthropicLLM:
        return self._model

sciphi/interface/anthropic_interface.py
emrgnt-cmplxty/sciphi
"""A module which defines interface abstractions for various LLM providers."""
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import List, Type

from sciphi.core import ProviderName
from sciphi.llm import LLM, LLMConfig, ModelName


class LLMInterface(ABC):
    """An abstract class to provide a common interface for LLM providers."""

    provider_name: ProviderName
    supported_models: list[ModelName] = []

    def __init__(
        self,
        config: LLMConfig,
    ) -> None:
        self.config = config

    @property
    @abstractmethod
    def model(self) -> LLM:
        """Property to get the instance of LLM."""
        pass

    @abstractmethod
    def get_completion(self, prompt: str) -> str:
        """Abstract method to get a completion from the provider."""
        pass

    def get_batch_completion(self, prompts: List[str]) -> List[str]:
        """Get a batch of completions from the provider."""
        return [self.get_completion(prompt) for prompt in prompts]


@dataclass
class ProviderConfig:
    """A dataclass to hold the configuration for a provider."""

    provider_name: ProviderName
    models: List[ModelName]
    llm_class: Type[LLMInterface]

sciphi/interface/base.py
emrgnt-cmplxty/sciphi
"""A module for interfacing with local HuggingFace models"""
import logging

from sciphi.interface.base import LLMInterface, ProviderName
from sciphi.interface.interface_manager import llm_provider
from sciphi.llm import HuggingFaceConfig, HuggingFaceLLM

logger = logging.getLogger(__name__)


@llm_provider
class HuggingFaceLLMInterface(LLMInterface):
    """A class to interface with local HuggingFace models."""

    provider_name = ProviderName.HUGGING_FACE

    def __init__(
        self,
        config: HuggingFaceConfig = HuggingFaceConfig(),
    ) -> None:
        self._model = HuggingFaceLLM(config)

    def get_completion(self, prompt: str) -> str:
        """Get a completion from the local HuggingFace provider."""

        logger.debug(
            f"Requesting completion from local HuggingFace with model={self._model.config.model_name} and prompt={prompt}"
        )
        return self.model.get_instruct_completion(prompt)

    @property
    def model(self) -> HuggingFaceLLM:
        return self._model

sciphi/interface/hugging_face_interface.py
emrgnt-cmplxty/sciphi
"""A module for managing provider interfaces."""
import logging
from typing import Type

from sciphi.interface.base import LLMInterface, ProviderConfig, ProviderName
from sciphi.llm import LLMConfig, ModelName

logger = logging.getLogger(__name__)


class InterfaceManager:
    """A class to manage provider interfaces."""

    provider_registry: dict[ProviderName, ProviderConfig] = {}

    @staticmethod
    def register_provider(
        provider: Type[LLMInterface],
    ) -> Type[LLMInterface]:
        """Register a provider with the InterfaceManager."""
        InterfaceManager.provider_registry[
            provider.provider_name
        ] = ProviderConfig(
            provider.provider_name, provider.supported_models, provider
        )
        return provider

    @staticmethod
    def get_provider(
        provider_name: ProviderName,
        model_name: str,
        config: LLMConfig,
        *args,
        **kwargs,
    ) -> LLMInterface:
        """Gets an interface based on the given provider and model name."""

        logger.debug(
            f"Loaded the following provider registry: {InterfaceManager.provider_registry}"
        )
        provider = InterfaceManager.provider_registry.get(provider_name)

        if not provider:
            raise ValueError(f"Provider '{provider_name}' not supported.")

        if provider.models and ModelName(model_name) not in provider.models:
            raise ValueError(
                f"Model '{model_name}' not supported by provider '{provider_name}'."
            )

        logger.info(
            f"Using provider '{provider_name}' with model '{model_name}' and configuration '{config}'."
        )
        return provider.llm_class(config, *args, **kwargs)


llm_provider = InterfaceManager.register_provider

sciphi/interface/interface_manager.py
emrgnt-cmplxty/sciphi
"""A module for interfacing with LiteLLM"""
import logging

from sciphi.interface.base import LLMInterface, ModelName
from sciphi.interface.interface_manager import llm_provider
from sciphi.llm import LiteLLM, LiteLLMConfig

logger = logging.getLogger(__name__)


@llm_provider
class LiteLLMInterface(LLMInterface):
    """A class to interface with the OpenAI API."""

    # import litellm
    instruct_models = [
        ModelName.GPT_3p5_TURBO_INSTRUCT,
    ]
    provider_name = "litellm"
    supported_models = []  # litellm.model_list

    system_message = "You are a helpful assistant."

    def __init__(
        self,
        config: LiteLLMConfig,
    ) -> None:
        self.config = config
        self._model = LiteLLM(config)

    def get_completion(self, prompt: str) -> str:
        """Get a completion from LiteLLM based on the provided prompt."""

        logger.debug(
            f"Getting completion from LiteLLM for model={self.config.model_name}"
        )
        if ModelName(self.config.model_name) in self.instruct_models:
            return self.model.get_instruct_completion(prompt)
        else:
            return self._model.get_chat_completion(
                [
                    {
                        "role": "system",
                        "content": LiteLLMInterface.system_message,
                    },
                    {"role": "user", "content": prompt},
                ]
            )

    @property
    def model(self) -> LiteLLM:
        return self._model

sciphi/interface/litellm_interface.py
emrgnt-cmplxty/sciphi
"""A module for interfacing with the LlamaIndex API"""
import logging

from sciphi.interface.base import LLMInterface, ProviderName
from sciphi.interface.interface_manager import llm_provider
from sciphi.llm import LLamaIndexConfig, LlamaIndexLLM

logger = logging.getLogger(__name__)


@llm_provider
class LlamaIndexInterface(LLMInterface):
    """A class to interface with the LlamaIndex API."""

    provider_name = ProviderName.LLAMA_INDEX

    def __init__(
        self,
        config: LLamaIndexConfig,
    ) -> None:
        super().__init__(config)
        if not isinstance(config, LLamaIndexConfig):
            raise ValueError(
                f"Expected an LLamaIndexConfig, but received a {type(config)}"
            )
        self.config: LLamaIndexConfig = config
        self._model: LlamaIndexLLM = LlamaIndexLLM(config)

    def get_completion(self, prompt: str) -> str:
        """Get a completion from the LlamaIndex API based on the provided prompt."""

        logger.debug(
            f"Getting completion from LlamaIndex API for model={self.config.model_name}, data_dir={self.config.llama_data_dir}, out_store_name={self.config.llama_out_store}, chunk_size={self.config.llama_chunk_size}, top_k_similarity={self.config.llama_top_k_similarity}, and prompt={prompt}"
        )
        return self.model.get_instruct_completion(prompt)

    @property
    def model(self) -> LlamaIndexLLM:
        return self._model

sciphi/interface/llama_index_interface.py
emrgnt-cmplxty/sciphi
"""A module for interfacing with the OpenAI API"""
import logging

from sciphi.interface.base import LLMInterface, ModelName, ProviderName
from sciphi.interface.interface_manager import llm_provider
from sciphi.llm import OpenAIConfig, OpenAILLM

logger = logging.getLogger(__name__)


@llm_provider
class OpenAILLMInterface(LLMInterface):
    """A class to interface with the OpenAI API."""

    instruct_models = [
        ModelName.GPT_3p5_TURBO_INSTRUCT,
    ]
    provider_name = ProviderName.OPENAI
    supported_models = [
        ModelName.GPT_3p5_TURBO_0301,
        ModelName.GPT_3p5_TURBO_0613,
        ModelName.GPT_3p5_TURBO,
        ModelName.GPT_3p5_TURBO_INSTRUCT,
        ModelName.GPT_3p5_TURBO_16k_0613,
        ModelName.GPT_4_0314,
        ModelName.GPT_4_0613,
        ModelName.GPT_4,
        ModelName.GPT_4_32k,
    ]
    system_message = "You are a helpful assistant."

    def __init__(
        self,
        config: OpenAIConfig,
    ) -> None:
        self.config = config
        self._model = OpenAILLM(config)

    def get_completion(self, prompt: str) -> str:
        """Get a completion from the OpenAI API based on the provided prompt."""

        logger.debug(
            f"Getting completion from OpenAI API for model={self.config.model_name}"
        )
        if ModelName(self.config.model_name) in self.instruct_models:
            return self.model.get_instruct_completion(prompt)
        else:
            return self._model.get_chat_completion(
                [
                    {
                        "role": "system",
                        "content": OpenAILLMInterface.system_message,
                    },
                    {"role": "user", "content": prompt},
                ]
            )

    @property
    def model(self) -> OpenAILLM:
        return self._model

sciphi/interface/openai_interface.py
emrgnt-cmplxty/sciphi
"""A module for interfacing with local vLLM models"""
import logging
from typing import List

from sciphi.interface.base import LLMInterface, ProviderName
from sciphi.interface.interface_manager import llm_provider
from sciphi.llm import vLLM, vLLMConfig

logger = logging.getLogger(__name__)


@llm_provider
class vLLMInterface(LLMInterface):
    """A class to interface with local vLLM models."""

    provider_name = ProviderName.VLLM

    def __init__(
        self,
        config: vLLMConfig = vLLMConfig(),
    ) -> None:
        self._model = vLLM(config)

    def get_completion(self, prompt: str) -> str:
        """Get a completion from the local vLLM provider."""

        logger.debug(
            f"Requesting completion from local vLLM with model={self._model.config.model_name} and prompt={prompt}"
        )
        return self.model.get_instruct_completion(prompt)

    def get_batch_completion(self, prompts: List[str]) -> List[str]:
        """Get a completion from the local vLLM provider."""

        logger.debug(
            f"Requesting completion from local vLLM with model={self._model.config.model_name} and prompts={prompts}"
        )
        return self.model.get_batch_instruct_completion(prompts)

    @property
    def model(self) -> vLLM:
        return self._model

sciphi/interface/vllm_interface.py
emrgnt-cmplxty/sciphi
from sciphi.llm.anthropic_llm import AnthropicConfig, AnthropicLLM
from sciphi.llm.base import LLM, LLMConfig, ModelName
from sciphi.llm.config_manager import LLMConfigManager
from sciphi.llm.hugging_face_llm import HuggingFaceConfig, HuggingFaceLLM
from sciphi.llm.lite_llm import LiteLLM, LiteLLMConfig
from sciphi.llm.llama_index_llm import LLamaIndexConfig, LlamaIndexLLM
from sciphi.llm.openai_llm import OpenAIConfig, OpenAILLM
from sciphi.llm.vllm_llm import vLLM, vLLMConfig

__all__ = [
    # Base
    "LLM",
    "ModelName",
    "LLMConfig",
    "LLMConfigManager",
    # Provider Models
    "AnthropicConfig",
    "AnthropicLLM",
    "HuggingFaceConfig",
    "HuggingFaceLLM",
    "LLamaIndexConfig",
    "LlamaIndexLLM",
    "OpenAIConfig",
    "OpenAILLM",
    "vLLMConfig",
    "vLLM",
    "LiteLLMConfig",
    "LiteLLM",
]

sciphi/llm/__init__.py
emrgnt-cmplxty/sciphi
"""A module for creating Anthropic models."""
from dataclasses import dataclass

from sciphi.core import ProviderName
from sciphi.llm.base import LLM, LLMConfig
from sciphi.llm.config_manager import model_config


@model_config
@dataclass
class AnthropicConfig(LLMConfig):
    """Configuration for Anthropic models."""

    # Base
    provider_name: ProviderName = ProviderName.ANTHROPIC
    model_name: str = "claude-2"
    temperature: float = 0.7
    top_p: float = 1.0

    # Anthropic Extras
    do_stream: bool = False
    max_tokens_to_sample: int = 256


class AnthropicLLM(LLM):
    """A concrete class for creating Anthropic models."""

    def __init__(
        self,
        config: AnthropicConfig,
    ) -> None:
        super().__init__(
            config,
        )
        try:
            from anthropic import AI_PROMPT, HUMAN_PROMPT, Anthropic
        except ImportError:
            raise ImportError(
                "Please install the anthropic package before attempting to run with an Anthropic model. This can be accomplished via `poetry install -E anthropic_support, ...OTHER_DEPENDENCIES_HERE`."
            )

        self.raw_prompt = HUMAN_PROMPT + " {instruction} " + AI_PROMPT
        self.anthropic = Anthropic()

        if not self.anthropic.api_key:
            raise ValueError(
                "Anthropic API key not found. Please set the ANTHROPIC_API_KEY environment variable."
            )

        # set the config here, again, for typing purposes
        if not isinstance(self.config, AnthropicConfig):
            raise ValueError(
                "The provided config must be an instance of AnthropicConfig."
            )
        self.config: AnthropicConfig = config

    def get_chat_completion(self, messages: list[dict[str, str]]) -> str:
        """Get a chat completion from the remote Anthropic API."""
        raise NotImplementedError(
            "Chat completion is not yet supported for Anthropic."
        )

    def get_instruct_completion(self, instruction: str) -> str:
        """Get an instruction completion from the remote Anthropic API."""

        formatted_prompt = self.raw_prompt.format(instruction=instruction)
        # TODO - Why does anthropic completion endpoint create a type error?
        completion = self.anthropic.completions.create(
            model=self.config.model_name,
            prompt=formatted_prompt,
            temperature=self.config.temperature,
            top_p=self.config.top_p,
            max_tokens_to_sample=self.config.max_tokens_to_sample,
            stream=self.config.do_stream,
        )  # type: ignore
        return completion.completion

sciphi/llm/anthropic_llm.py
emrgnt-cmplxty/sciphi
"""Base classes for language model providers."""
from abc import ABC, abstractmethod
from dataclasses import dataclass, fields
from enum import Enum

from sciphi.core import ProviderName


class ModelName(Enum):
    """An enum to hold the names of supported models."""

    # OpenAI Models

    ## GPT-3.5
    GPT_3p5_TURBO_0301 = "gpt-3.5-turbo-0301"
    GPT_3p5_TURBO_0613 = "gpt-3.5-turbo-0613"
    GPT_3p5_TURBO_16k_0613 = "gpt-3.5-turbo-16k-0613"
    GPT_3p5_TURBO = "gpt-3.5-turbo"
    GPT_3p5_TURBO_INSTRUCT = "gpt-3.5-turbo-instruct"

    ## GPT-4
    GPT_4_0314 = "gpt-4-0314"
    GPT_4_0613 = "gpt-4-0613"
    GPT_4 = "gpt-4"
    GPT_4_32k = "gpt-4-32k"

    # Anthropic Models

    CLAUDE_INSTANT_1 = "claude-instant-1"
    CLAUDE_2 = "claude-2"


@dataclass
class LLMConfig(ABC):
    provider_name: ProviderName
    model_name: str
    temperature: float
    top_p: float

    version: str = "0.1.0"

    @classmethod
    def create(cls, **kwargs):
        valid_keys = {f.name for f in fields(cls)}
        filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_keys}
        return cls(**filtered_kwargs)


class LLM(ABC):
    """An abstract class to provide a common interface for LLMs."""

    def __init__(
        self,
        config: LLMConfig,
    ) -> None:
        self.config = config

    @abstractmethod
    def get_chat_completion(self, messages: list[dict[str, str]]) -> str:
        """Abstract method to get a chat completion from the provider."""
        pass

    @abstractmethod
    def get_instruct_completion(self, instruction: str) -> str:
        """Abstract method to get an instruction completion from the provider."""
        pass

sciphi/llm/base.py
emrgnt-cmplxty/sciphi
"""A module to manage the configurations for models."""
from typing import Type

from sciphi.core import ProviderName
from sciphi.llm.base import LLMConfig


class LLMConfigManager:
    """A class to manage the configurations for models."""

    config_registry: dict[ProviderName, Type[LLMConfig]] = {}

    @staticmethod
    def get_config_for_provider(
        provider_name: ProviderName,
    ) -> Type[LLMConfig]:
        """Get the configuration class for a given model."""
        config_class = LLMConfigManager.config_registry.get(provider_name)

        if not config_class:
            raise ValueError(
                f"No provider configuration found for provider '{provider_name}'."
            )

        return config_class

    @staticmethod
    def register_config(
        config_cls: Type[LLMConfig],
    ) -> Type[LLMConfig]:
        """Register a provider with the LLMConfigManager."""
        LLMConfigManager.config_registry[config_cls.provider_name] = config_cls
        return config_cls


model_config = LLMConfigManager.register_config

sciphi/llm/config_manager.py
emrgnt-cmplxty/sciphi
"""A module for managing local HuggingFace models."""

from dataclasses import dataclass, field

from sciphi.core import ProviderName
from sciphi.llm.base import LLM, LLMConfig
from sciphi.llm.config_manager import model_config


@model_config
@dataclass
class HuggingFaceConfig(LLMConfig):
    """Configuration for local HuggingFace models."""

    # Base
    provider_name: ProviderName = ProviderName.HUGGING_FACE
    model_name: str = "gpt2"
    temperature: float = 0.7
    top_p: float = 1.0

    # Generation parameters
    top_k: int = 100
    max_tokens_to_sample: int = 256
    do_sample: bool = True
    num_beams: int = 1

    # Model and Tokenizer extras
    device: str = "cpu"
    add_model_kwargs: dict = field(default_factory=dict)
    add_tokenizer_kwargs: dict = field(default_factory=dict)
    add_generation_kwargs: dict = field(default_factory=dict)


class HuggingFaceLLM(LLM):
    """A concrete class for creating a local HuggingFace model."""

    def __init__(
        self,
        config: HuggingFaceConfig,
    ) -> None:
        try:
            import torch  # noqa: F401
            from transformers import (
                AutoModelForCausalLM,
                AutoTokenizer,
                GenerationConfig,
            )
        except ImportError:
            raise ImportError(
                "Please install the torch and transformers packages before attempting to run with a HuggingFace model. This can be accomplished via `poetry install -E hf_support, ...OTHER_DEPENDENCIES_HERE`."
            )

        super().__init__(
            config,
        )
        # Set the config here, again, for typing purposes
        # TODO - Can setting the config twice be avoided?
        if not isinstance(self.config, HuggingFaceConfig):
            raise ValueError(
                "The provided config must be an instance of HuggingFaceConfig."
            )
        self.config: HuggingFaceConfig = config

        # Create the model, tokenizer, and generation config
        model_name = self.config.model_name

        self.model = AutoModelForCausalLM.from_pretrained(
            model_name,
            device_map=self.config.device,
            trust_remote_code=True,
            **config.add_model_kwargs,
        )
        self.tokenizer = AutoTokenizer.from_pretrained(
            model_name,
            device_map=self.config.device,
            **config.add_tokenizer_kwargs,
        )
        self.generation_config = GenerationConfig.from_pretrained(
            model_name,
            top_k=self.config.top_k,
            top_p=self.config.top_p,
            max_new_tokens=self.config.max_tokens_to_sample,
            do_sample=self.config.do_sample,
            num_beams=self.config.num_beams,
            **config.add_generation_kwargs,
        )

    def get_chat_completion(self, messages: list[dict[str, str]]) -> str:
        """Get a completion from the local HuggingFace model."""
        raise NotImplementedError(
            "Chat completion not yet implemented for HuggingFace."
        )

    def get_instruct_completion(self, instruction: str) -> str:
        """Get an instr. completion from local HuggingFace model."""
        # TODO - Should we set `max_length` here? What about `max_tokens_to_sample`?
        # Should we set the device on inputs or is this handled above? Test on GPU inst.
        inputs = self.tokenizer(
            instruction,
            return_tensors="pt",
        ).to(self.config.device)
        raw_completion = self.model.generate(
            inputs["input_ids"], generation_config=self.generation_config
        )
        return self.tokenizer.batch_decode(raw_completion)[0].replace(
            instruction, ""
        )

sciphi/llm/hugging_face_llm.py
emrgnt-cmplxty/sciphi
"""A module for creating OpenAI model abstractions."""

from dataclasses import dataclass
from typing import Optional

import tiktoken

from sciphi.core import ProviderName
from sciphi.llm.base import LLM, LLMConfig
from sciphi.llm.config_manager import model_config


@model_config
@dataclass
class LiteLLMConfig(LLMConfig):
    """Configuration for LiteLLM models."""

    # Base
    provider_name: ProviderName = ProviderName.OPENAI
    model_name: str = "gpt-3.5-turbo"
    temperature: float = 0.1
    top_p: float = 1.0

    # OpenAI Extras
    do_stream: bool = False
    max_tokens_to_sample: int = 256
    max_total_tokens = 4_096  # TODO - automate population of this.
    functions: Optional[list[dict]] = None


class LiteLLM(LLM):
    """A concrete class for creating LiteLLM models."""

    def __init__(
        self,
        config: LiteLLMConfig,
    ) -> None:
        super().__init__(config)
        try:
            import litellm
        except ImportError:
            raise ImportError(
                "Please install the litellm package before attempting to run with a LiteLLM model. This can be accomplished via `poetry add litellm`."
            )
        # set the config here, again, for typing purposes
        if not isinstance(self.config, LiteLLMConfig):
            raise ValueError(
                "The provided config must be an instance of LiteLLMConfig."
            )
        self.config: LiteLLMConfig = config

    def get_chat_completion(self, messages: list[dict[str, str]]) -> str:
        """Get a completion from the LiteLLM API based on the provided messages."""
        import litellm

        # Create a dictionary with the default arguments
        args = self._get_base_args()

        args["messages"] = messages

        # Conditionally add the 'functions' argument if it's not None
        if self.config.functions is not None:
            args["functions"] = self.config.functions

        # Create the chat completion
        response = litellm.completion(**args)
        return response.choices[0].message["content"]

    def get_instruct_completion(self, instruction: str) -> str:
        """Get an instruction completion from the LiteLLM based on the provided prompt."""
        import litellm

        # Create a dictionary with the default arguments
        args = self._get_base_args(instruction)

        # Create the instruction completion
        args["prompt"] = instruction
        response = litellm.completion(**args)
        return response.choices[0].text

    def _get_base_args(
        self,
        prompt=None,
    ) -> dict:
        """Get the base arguments for the OpenAI API."""
        encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")

        return {
            "model": self.config.model_name,
            "temperature": self.config.temperature,
            "top_p": self.config.top_p,
            "max_tokens": self.config.max_total_tokens
            - len(encoding.encode(prompt))
            if prompt
            else self.config.max_tokens_to_sample,
            "stream": self.config.do_stream,
        }

sciphi/llm/lite_llm.py
emrgnt-cmplxty/sciphi
"""A module for creating LlamaIndex models."""
import os
from dataclasses import dataclass

from sciphi.core.utils import get_data_raw_dir
from sciphi.interface.base import ProviderName
from sciphi.llm.base import LLM
from sciphi.llm.config_manager import model_config
from sciphi.llm.openai_llm import OpenAIConfig


@model_config
@dataclass
class LLamaIndexConfig(OpenAIConfig):
    """A class to manage the configurations for LlamaIndex."""

    provider_name: ProviderName = ProviderName.LLAMA_INDEX

    # LlamaIndex-specific configs
    # Defaults to the algorithms textbook
    llama_data_dir: str = os.path.join(get_data_raw_dir(), "algorithms")
    llama_index_name: str = "vector_index"
    llama_out_store: str = "storage"
    llama_load_from_avail_store: bool = True
    llama_chunk_size: int = 512
    llama_top_k_similarity: int = 5


class LlamaIndexLLM(LLM):
    """A concrete class for creating LlamaIndex models."""

    def __init__(
        self,
        config: LLamaIndexConfig,
    ) -> None:
        super().__init__(config)
        try:
            import openai
            from llama_index import (
                OpenAIEmbedding,
                ServiceContext,
                SimpleDirectoryReader,
                StorageContext,
                VectorStoreIndex,
                load_index_from_storage,
            )
            from llama_index.llms import OpenAI
        except ImportError:
            raise ImportError(
                "Please install the llama_index package before attempting to run with a LlamaIndex. This can be accomplished via `poetry install -E llama_index_support, ...OTHER_DEPENDENCIES_HERE`."
            )
        if not openai.api_key:
            raise ValueError(
                "OpenAI API key not found. Please set the OPENAI_API_KEY environment variable."
            )
        self.config: LLamaIndexConfig = config

        llama_index_llm = OpenAI(
            model=self.config.model_name,
            temperature=self.config.temperature,
            max_tokens=self.config.max_tokens_to_sample,
        )
        embed_model = OpenAIEmbedding()

        service_context = ServiceContext.from_defaults(
            llm=llama_index_llm,
            embed_model=embed_model,
        )

        store_dir = os.path.join(
            self.config.llama_data_dir, self.config.llama_out_store
        )
        store_dir_exists = os.path.exists(store_dir)
        loaded_store = False
        if not self.config.llama_load_from_avail_store or not store_dir_exists:
            # Creating a new store
            # Load Documents
            documents = SimpleDirectoryReader(
                self.config.llama_data_dir
            ).load_data()

            # Create index with specified chunk size
            index = VectorStoreIndex.from_documents(
                documents, chunk_size=self.config.llama_chunk_size
            )
        else:
            # Loading from an existing store
            storage_context = StorageContext.from_defaults(
                persist_dir=store_dir
            )

            index: VectorStoreIndex = load_index_from_storage(  # type: ignore
                storage_context, index_id=self.config.llama_index_name
            )
            loaded_store = True

        # Create a query engine with the retriever and response synthesizer
        self.query_engine = index.as_query_engine(
            similarity_top_k=self.config.llama_top_k_similarity,
            service_context=service_context,
        )
        # Save index to disk if it was just created
        # TODO - Add some upstream customization around this.
        if not loaded_store:
            index.set_index_id(self.config.llama_index_name)
            index.storage_context.persist(store_dir)

    def get_chat_completion(self, messages: list[dict[str, str]]) -> str:
        """Get a completion from the LlamaIndex API based on the provided messages."""
        raise NotImplementedError(
            "Chat completion is not yet supported for LlamaIndex."
        )

    def get_instruct_completion(self, instruction: str) -> str:
        """Get an instruction completion from the LlamaIndex API based on the provided prompt."""
        # Clean the typical instruction completion for better search results
        if "### Instruction:" in instruction:
            instruction = instruction.split("### Instruction:")[1].strip()
        if "### Response:" in instruction:
            instruction = instruction.replace("### Response:", "").strip()
        return self.query_engine.query(instruction).response  # type: ignore

sciphi/llm/llama_index_llm.py
emrgnt-cmplxty/sciphi
"""A module for creating OpenAI model abstractions."""

from dataclasses import dataclass
from typing import Optional

import tiktoken

from sciphi.core import ProviderName
from sciphi.llm.base import LLM, LLMConfig
from sciphi.llm.config_manager import model_config


@model_config
@dataclass
class OpenAIConfig(LLMConfig):
    """Configuration for OpenAI models."""

    # Base
    provider_name: ProviderName = ProviderName.OPENAI
    model_name: str = "gpt-3.5-turbo"
    temperature: float = 0.1
    top_p: float = 1.0

    # OpenAI Extras
    do_stream: bool = False
    max_tokens_to_sample: int = 256
    max_total_tokens = 4_096  # TODO - automate population of this.
    functions: Optional[list[dict]] = None


class OpenAILLM(LLM):
    """A concrete class for creating OpenAI models."""

    def __init__(
        self,
        config: OpenAIConfig,
    ) -> None:
        super().__init__(config)
        try:
            import openai
        except ImportError:
            raise ImportError(
                "Please install the openai package before attempting to run with an OpenAI model. This can be accomplished via `poetry install -E openai_support, ...OTHER_DEPENDENCIES_HERE`."
            )
        if not openai.api_key:
            raise ValueError(
                "OpenAI API key not found. Please set the OPENAI_API_KEY environment variable."
            )
        # set the config here, again, for typing purposes
        if not isinstance(self.config, OpenAIConfig):
            raise ValueError(
                "The provided config must be an instance of OpenAIConfig."
            )
        self.config: OpenAIConfig = config

    def get_chat_completion(self, messages: list[dict[str, str]]) -> str:
        """Get a completion from the OpenAI API based on the provided messages."""
        import openai

        # Create a dictionary with the default arguments
        args = self._get_base_args()

        args["messages"] = messages

        # Conditionally add the 'functions' argument if it's not None
        if self.config.functions is not None:
            args["functions"] = self.config.functions

        # Create the chat completion
        response = openai.ChatCompletion.create(**args)
        return response.choices[0].message["content"]

    def get_instruct_completion(self, instruction: str) -> str:
        """Get an instruction completion from the OpenAI API based on the provided prompt."""
        import openai

        # Create a dictionary with the default arguments
        args = self._get_base_args(instruction)

        # Create the instruction completion
        args["prompt"] = instruction
        response = openai.Completion.create(**args)
        return response.choices[0].text

    def _get_base_args(
        self,
        prompt=None,
    ) -> dict:
        """Get the base arguments for the OpenAI API."""
        encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")

        return {
            "model": self.config.model_name,
            "temperature": self.config.temperature,
            "top_p": self.config.top_p,
            "max_tokens": self.config.max_total_tokens
            - len(encoding.encode(prompt))
            if prompt
            else self.config.max_tokens_to_sample,
            "stream": self.config.do_stream,
        }

sciphi/llm/openai_llm.py
emrgnt-cmplxty/sciphi
"""A module for managing local vLLM models."""

from dataclasses import dataclass

from sciphi.core import ProviderName
from sciphi.llm.base import LLM, LLMConfig
from sciphi.llm.config_manager import model_config


@model_config
@dataclass
class vLLMConfig(LLMConfig):
    """Configuration for local vLLM models."""

    # Base
    provider_name: ProviderName = ProviderName.VLLM
    model_name: str = "gpt2"
    temperature: float = 0.7
    top_p: float = 1.0

    # Generation parameters
    top_k: int = 100
    max_tokens_to_sample: int = 256


class vLLM(LLM):
    """Configuration for local vLLM models."""

    def __init__(
        self,
        config: vLLMConfig,
    ) -> None:
        super().__init__(config)
        try:
            from vllm import LLM as vvLLM
            from vllm import SamplingParams
        except ImportError:
            raise ImportError(
                "Please install the vllm package before attempting to run with an vLLM model. This can be accomplished via `poetry install -E vllm_support, ...OTHER_DEPENDENCIES_HERE`."
            )
        self.model = vvLLM(model=config.model_name)
        self.sampling_params = SamplingParams(
            temperature=config.temperature,
            top_p=config.top_p,
            top_k=config.top_k,
            max_tokens=config.max_tokens_to_sample,
        )

    def get_chat_completion(self, messages: list[dict[str, str]]) -> str:
        """Get a completion from the OpenAI API based on the provided messages."""
        raise NotImplementedError(
            "Chat completion not yet implemented for vLLM."
        )

    def get_instruct_completion(self, prompt: str) -> str:
        """Get an instruction completion from local vLLM API."""
        outputs = self.model.generate([prompt], self.sampling_params)
        return outputs[0].outputs[0].text

    def get_batch_instruct_completion(self, prompts: list[str]) -> list[str]:
        """Get batch instruction completion from local vLLM."""
        raw_outputs = self.model.generate(prompts, self.sampling_params)
        return [ele.outputs[0].text for ele in raw_outputs]

sciphi/llm/vllm_llm.py
emrgnt-cmplxty/sciphi
from sciphi.makers.data_maker import DataMaker

__all__ = ["DataMaker"]

sciphi/makers/__init__.py
emrgnt-cmplxty/sciphi
"""A module which facilitates synthesizing prompt data."""

from copy import copy
from typing import Generator, Optional

from sciphi.config.base import DataGeneratorMode
from sciphi.prompt import Prompt, PromptGenerator


class DataMaker:
    """A class to synthesize data from a configuration."""

    PROMPT_TEMPLATE_TAG = "prompt_templates"

    def __init__(
        self,
        generator_mode: DataGeneratorMode,
        prompt_generator: PromptGenerator,
        outer_prompt: Prompt,
        dataset_name: Optional[str] = None,
        dataset_filters: Optional[dict[str, str]] = None,
    ) -> None:
        self.generator_mode = generator_mode
        self.prompt_generator = prompt_generator
        self.outer_prompt = outer_prompt
        self.dataset_name = dataset_name
        self.dataset_filters = dataset_filters

    def synthetic_generator(
        self, batch_size: int, num_samples: int
    ) -> Generator[list[str], None, None]:
        """Returns a generator which yields formatted prompts from the loaded configuration."""

        counter = 0
        while counter < num_samples:
            batch: list[str] = []

            while len(batch) < batch_size:
                inner_prompt = self.prompt_generator.generate_prompt()
                formatted_outer_prompt = copy(self.outer_prompt)
                formatted_outer_prompt.format(
                    instruction=inner_prompt[
                        PromptGenerator.FORMATTED_PROMPT_TAG
                    ]
                )
                batch.append(formatted_outer_prompt.text)
                counter += 1
            yield batch

    def hf_dataset_generator(
        self,
        batch_size: int,
        num_samples: int,
    ) -> Generator[list[str], None, None]:
        """Returns a generator which yields formatted prompts from the loaded configuration."""
        if batch_size != 1:
            raise ValueError(
                "Batch size must be 1 for HuggingFace dataset generation."
            )

        try:
            from datasets import load_dataset
        except ImportError:
            raise ImportError(
                "Please install the `datasets` package before attempting to run with a HuggingFace dataset generator. This can be accomplished via `poetry install -E hf_support, ...OTHER_DEPENDENCIES_HERE`."
            )

        dataset = load_dataset(self.dataset_name, streaming=True)

        # filter the dataset, on key and list of filter values
        if self.dataset_filters:
            for key, values in self.dataset_filters.items():
                dataset = dataset.filter(
                    lambda example: example[key] in values,
                )
        counter = 0
        for data in dataset["train"]:
            inner_prompt = self.prompt_generator.generate_prompt(
                optional_formatters=data
            )
            formatted_outer_prompt = copy(self.outer_prompt)
            formatted_outer_prompt.format(
                instruction=inner_prompt[PromptGenerator.FORMATTED_PROMPT_TAG]
            )
            yield [formatted_outer_prompt.text]
            counter += 1
            if counter >= num_samples:
                break

    def generator(
        self, batch_size=1_024, num_samples=1_048_576
    ) -> Generator[list[str], None, None]:
        """Returns a generator which yields formatted prompts from the loaded configuration."""

        if self.generator_mode == DataGeneratorMode.SYNTHETIC:
            yield from self.synthetic_generator(batch_size, num_samples)
        elif self.generator_mode == DataGeneratorMode.FROM_HF_DATASET:
            yield from self.hf_dataset_generator(batch_size, num_samples)
        else:
            raise ValueError(
                f"Invalid generation mode {self.generator_mode} specified. Must be one of {DataGeneratorMode}."
            )

sciphi/makers/data_maker.py
emrgnt-cmplxty/sciphi
from sciphi.prompt.base import Prompt, PromptManager, PromptStructure
from sciphi.prompt.generator import PromptGenerator

__all__ = ["Prompt", "PromptManager", "PromptStructure", "PromptGenerator"]

sciphi/prompt/__init__.py
emrgnt-cmplxty/sciphi
"""A module to facilitate seamless construction of input prompts"""
from dataclasses import dataclass
from enum import Enum
from typing import Optional, Union


class PromptStructure(Enum):
    """How are prompts formatted (e.g. single reply, conversation, ..)"""

    SINGLE = "single"
    CONVERSATION = "conversation"


@dataclass
class Prompt:
    """A simple abstraction for building input prompts."""

    expected_inputs: set[str]
    raw_text: Union[str, list[str]]
    structure: PromptStructure
    _text: Optional[Union[str, list[str]]] = None

    def format(self, **kwargs) -> None:
        """Format the prompt into a string"""
        if set(kwargs.keys()) != self.expected_inputs:
            raise ValueError(
                f"Received {kwargs.keys()}, but expected {self.expected_inputs}"
            )
        if self.structure == PromptStructure.SINGLE:
            if not isinstance(self.raw_text, str):
                raise ValueError(
                    f"Expected `raw_text` to be a string, but got {type(self.raw_text)}"
                )
            self._text = self.raw_text.format(**kwargs)
        else:
            if not isinstance(self.raw_text, list) and all(
                isinstance(item, str) for item in self.raw_text
            ):
                raise ValueError(
                    f"Expected `raw_text` to be a list of strings, but got {type(self.raw_text)}"
                )
            # If not a single response, then text is formatted in a list
            self._text = [ele.format(**kwargs) for ele in self.raw_text]

    @property
    def text(self):
        if not self._text:
            raise ValueError(
                "Format prompt before attempting to access the `text` field."
            )
        return self._text


class PromptManager:
    """A manager for available prompts"""

    stock_prompts = {
        "md_instruction": Prompt(
            expected_inputs={"instruction"},
            raw_text="""Below is an instruction that describes a task. Write a response that appropriately completes the request.\n### Instruction:\n{instruction}\n\n### Response:""",
            structure=PromptStructure.SINGLE,
        ),
        "unit": Prompt(
            expected_inputs={"instruction"},
            raw_text="""{instruction}""",
            structure=PromptStructure.SINGLE,
        ),
    }

    def __init__(self, stock_prompts=stock_prompts) -> None:
        self.stock_prompts = stock_prompts

    def get_prompt(self, selected_prompt: str) -> Prompt:
        """
        Gets the list of prompts associated with the specified mode
        """
        prompt = self.stock_prompts.get(selected_prompt)
        if not prompt:
            raise ValueError(
                f"Requested prompt mode {selected_prompt} does not exist!"
            )
        return prompt

sciphi/prompt/base.py
emrgnt-cmplxty/sciphi
"""A module to manage prompt generation for a given configuration."""
import random
from typing import Union


class PromptGenerator:
    """A class for generating prompts from a data config."""

    RAW_PROMPT_TAG = "raw_prompt"
    FORMATTED_PROMPT_TAG = "formatted_prompt"

    # TODO - Add type restrictions for config
    def __init__(
        self,
        config: dict,
        prompt_templates: dict[str, int],
        prompt_template_input_dependencies: dict[str, str],
        prompt_dataset_dependencies: dict[str, str],
        prompt_inputs: list[str],
    ):
        self.config = config
        self.prompt_templates = prompt_templates
        self.prompt_template_input_dependencies = (
            prompt_template_input_dependencies
        )
        self.prompt_inputs = prompt_inputs
        self.prompt_dataset_dependencies = prompt_dataset_dependencies

    @staticmethod
    def _random_sample(vars_and_weights: dict) -> str:
        """Randomly sample a weighted dictionary."""

        keys, weights = zip(*vars_and_weights.items())
        if len(keys) == 0:
            raise IndexError("Cannot randomly sample an empty input.")
        return random.choices(keys, weights)[0]

    def generate_prompt(self, optional_formatters=None) -> dict:
        """Return a prompt and its inputs."""
        # Build the prompt formatters
        formatters: dict[str, str] = optional_formatters or {}

        # self.prompt_input_dependencies is a dictionary of dictionaries
        # The outer dictionary is keyed by the prompt input
        # The inner dictionary is keyed by the prompt input dependency
        # It can be `None` if there are no dependencies
        for prompt_input in self.prompt_inputs:
            entry = self.config[prompt_input]
            if (
                self.prompt_dataset_dependencies
                and prompt_input in self.prompt_dataset_dependencies
            ):
                # Parse dataset dependencies
                entry = PromptGenerator._random_sample(entry)

                self._insert_formatter(
                    formatters, prompt_input, optional_formatters[entry]
                )
            elif (
                self.prompt_template_input_dependencies
                and prompt_input in self.prompt_template_input_dependencies
            ):
                # Parse single depth dependencies
                dependent_on = self.prompt_template_input_dependencies[
                    prompt_input
                ]
                dependent_value = formatters[dependent_on]
                self._insert_formatter(
                    formatters,
                    prompt_input,
                    entry[dependent_value],
                )
            else:
                self._insert_formatter(formatters, prompt_input, entry)

        prompt = PromptGenerator._random_sample(self.prompt_templates)
        return {
            PromptGenerator.RAW_PROMPT_TAG: prompt,
            PromptGenerator.FORMATTED_PROMPT_TAG: prompt.format_map(
                formatters
            ),
        }

    @staticmethod
    def _insert_formatter(
        formatters: dict, key: str, entry: Union[dict, str]
    ) -> None:
        """Insert a formatter into the formatters dictionary."""
        if isinstance(entry, dict):
            formatters[key] = PromptGenerator._random_sample(entry)
        elif isinstance(entry, str):
            formatters[key] = entry
        else:
            raise ValueError(
                f"Unexpected type for config entry {key}: {type(entry)}"
            )

sciphi/prompt/generator.py
emrgnt-cmplxty/sciphi
from sciphi.writers.jsonl_writer import JsonlDataWriter
from sciphi.writers.raw_writer import RawDataWriter

__all__ = ["JsonlDataWriter", "RawDataWriter"]

sciphi/writers/__init__.py
emrgnt-cmplxty/sciphi
"""A module which defines the abstract bata class for data writers."""
import os
import time
from abc import ABC, abstractmethod


class DataWriter(ABC):
    """An abstract class for data writers."""

    def __init__(self, output_path):
        self.output_path = output_path

    def _get_modified_path(self):
        """Modify the output path if the file already exists and overwriting is not allowed."""
        if not self.overwrite and os.path.exists(self.output_path):
            base_name, ext = os.path.splitext(self.output_path)
            return f"{base_name}_{int(time.time())}{ext}"
        return self.output_path

    @abstractmethod
    def write(self, data):
        """Write data to the specified path."""
        pass

sciphi/writers/base.py
emrgnt-cmplxty/sciphi
"""A module which facilitates JSONL data writing."""
import json

from sciphi.writers.base import DataWriter


class JsonlDataWriter(DataWriter):
    """A class to write data to a JSONL file."""

    def __init__(self, output_path, overwrite=True):
        """Initialize the DataWriter."""
        self.output_path = output_path
        self.overwrite = overwrite

    def write(self, data: list[dict]) -> None:
        """
        Write the provided data to the specified path.

        Args:
            data (list): List of data entries to be written.
        """
        path = self._get_modified_path()

        with open(path, "a") as f:
            for entry in data:
                f.write(json.dumps(entry) + "\n")

sciphi/writers/jsonl_writer.py
emrgnt-cmplxty/sciphi
"""A module which facilitates raw data writing."""
from sciphi.writers.base import DataWriter


class RawDataWriter(DataWriter):
    """A class to write raw data file."""

    def __init__(self, output_path, overwrite=True):
        """Initialize the DataWriter."""
        self.output_path = output_path
        self.overwrite = overwrite

    def write(self, data: str) -> None:
        """
        Write the provided data to the specified path.

        Args:
            data (list): List of data entries to be written.
        """
        path = self._get_modified_path()

        with open(path, "a") as f:
            f.write(data + "\n")

sciphi/writers/raw_writer.py
emrgnt-cmplxty/sciphi