A Stata emulator in Python with support for machine learning and LLM fine tuning
# content of conftest.py
# See https://docs.pytest.org/en/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option
import pytest


def pytest_addoption(parser):
    parser.addoption(
        "--runslow", action="store_true", default=False, help="run slow tests"
    )


def pytest_configure(config):
    config.addinivalue_line("markers", "slow: mark test as slow to run")


def pytest_collection_modifyitems(config, items):
    if config.getoption("--runslow"):
        # --runslow given in cli: do not skip slow tests
        return
    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
    for item in items:
        if "slow" in item.keywords:
            item.add_marker(skip_slow)

conftest.py
pandichef/pdexplorer
from pprint import pprint, pformat

# import pandas as pd
# import ._command_abbreviation # type: ignore
from .regress import regress as reg
from .regress import regress as regr
from .regress import regress as regre
from .regress import regress as regres

from .regress import regress as regress

from .nn.regressnn import regressnn
from .ml.regressml import regressml
from .ml.treeclassify import treeclassify as tree
from .ml.treeclassify import treeclassify as treec
from .ml.treeclassify import treeclassify as treecl
from .ml.treeclassify import treeclassify as treecla
from .ml.treeclassify import treeclassify as treeclas
from .ml.treeclassify import treeclassify as treeclass
from .ml.treeclassify import treeclassify as treeclassi
from .ml.treeclassify import treeclassify as treeclassif
from .ml.treeclassify import treeclassify as treeclassify

from .webuse import webuse
from .sysuse import sysuse
from .logit import logit
from .drop import drop

from .browse import browse as br
from .browse import browse as bro
from .browse import browse as brow
from .browse import browse as brows
from .browse import browse as browse
from .shortcuts.dfedit import dfedit as dfed
from .shortcuts.dfedit import dfedit as dfedi
from .shortcuts.dfedit import dfedit as dfedit
from .gsort import gsort
from .order import order
from .keep import keep

from .finance.mtgyield import mtgyield
from .finance.mtgprice import mtgprice
from .plus.srecode import srecode
from .shortcuts.profile import profile
from .doedit import doedit as doed
from .doedit import doedit as doedi
from .doedit import doedit as doedit
from .doedit import doedit as pyed
from .doedit import doedit as pyedi
from .doedit import doedit as pyedit

# from .scatter import scatter as scatter_old
from .xpose import xpose
from .lis import lis as li
from .lis import lis as lis

from .clear import clear
from .clear import clearall
from .use import use

# from .keepin import keepin
# from .keepif import keepif
# from .dropif import dropif
# from .dropin import dropin
from .generate import generate as gen
from .generate import generate as gene
from .generate import generate as gener
from .generate import generate as genera
from .generate import generate as generat
from .generate import generate as generate
from .replace import replace
from .summarize import summarize as sum  # "describe" in pandas
from .summarize import summarize as summ
from .summarize import summarize as summa
from .summarize import summarize as summar
from .summarize import summarize as summari
from .summarize import summarize as summariz
from .summarize import summarize as summarize
from .describe import describe  # "info" in pandas
from .tabulate import tabulate as tab
from .tabulate import tabulate as tabu
from .tabulate import tabulate as tabul
from .tabulate import tabulate as tabula
from .tabulate import tabulate as tabulat
from .tabulate import tabulate as tabulate
from .rename import rename
from .save import save
from .preserve import preserve
from .merge import merge
from .restore import restore
from ._quietly import quietly
from .collapse import collapse
from .egen import egen
from .reshape import reshapelong, reshapewide
from .sort import sort
from ._by import by  # context manager
from .shortcuts.melt import melt
from ._print_horizontal_line import print_horizontal_line
from .cf import cf
from .returnlist import (
    returnlist,
    r_,
    ereturnlist,
    e_,
    sreturnlist,
    s_,
    nreturnlist,
    n_,
    creturnlist,
    c_,
)


# from ._dataset import Dataset
# from ._dataset import current
from pdexplorer._dataset import current


def methods():
    # global singleton
    for k, v in current.methods.items():
        print(k + ": ", v)


def properties():
    pprint(current.properties)


# def describe():
#     singleton.df.info()


# props = pformat(singleton.properties)

# from pdexplorer._altair_mapper import circlechart_ as circle_  # type: ignore

# print(_circlechart)
# print(_circlechart)
# print(_circlechart)
# print(_circlechart)
#  as blahblah  # type: ignore

# _circle = blahblah

from ._altair_mapper import (
    arcchart as arc,  # type: ignore
    arcchart as arcc,  # type: ignore
    arcchart as arcch,  # type: ignore
    arcchart as arccha,  # type: ignore
    arcchart as arcchar,  # type: ignore
    arcchart as arcchart,  # type: ignore
    areachart as area,  # type: ignore
    areachart as areac,  # type: ignore
    areachart as areach,  # type: ignore
    areachart as areacha,  # type: ignore
    areachart as areachar,  # type: ignore
    areachart as areachart,  # type: ignore
    barchart as bar,  # type: ignore
    barchart as barc,  # type: ignore
    barchart as barch,  # type: ignore
    barchart as barcha,  # type: ignore
    barchart as barchar,  # type: ignore
    barchart as barchart,  # type: ignore
    circlechart as circle,  # type: ignore
    circlechart as circlec,  # type: ignore
    circlechart as circlech,  # type: ignore
    circlechart as circlecha,  # type: ignore
    circlechart as circlechar,  # type: ignore
    circlechart as circlechart,  # type: ignore
    geoshapechart as geoshape,  # type: ignore
    geoshapechart as geoshapec,  # type: ignore
    geoshapechart as geoshapech,  # type: ignore
    geoshapechart as geoshapecha,  # type: ignore
    geoshapechart as geoshapechar,  # type: ignore
    geoshapechart as geoshapechart,  # type: ignore
    imagechart as image,  # type: ignore
    imagechart as imagec,  # type: ignore
    imagechart as imagech,  # type: ignore
    imagechart as imagecha,  # type: ignore
    imagechart as imagechar,  # type: ignore
    imagechart as imagechart,  # type: ignore
    linechart as line,  # type: ignore
    linechart as linec,  # type: ignore
    linechart as linech,  # type: ignore
    linechart as linecha,  # type: ignore
    linechart as linechar,  # type: ignore
    linechart as linechart,  # type: ignore
    pointchart as point,  # type: ignore
    pointchart as pointc,  # type: ignore
    pointchart as pointch,  # type: ignore
    pointchart as pointcha,  # type: ignore
    pointchart as pointchar,  # type: ignore
    pointchart as pointchart,  # type: ignore
    rectchart as rect,  # type: ignore
    rectchart as rectc,  # type: ignore
    rectchart as rectch,  # type: ignore
    rectchart as rectcha,  # type: ignore
    rectchart as rectchar,  # type: ignore
    rectchart as rectchart,  # type: ignore
    rulechart as rule,  # type: ignore
    rulechart as rulec,  # type: ignore
    rulechart as rulech,  # type: ignore
    rulechart as rulecha,  # type: ignore
    rulechart as rulechar,  # type: ignore
    rulechart as rulechart,  # type: ignore
    squarechart as square,  # type: ignore
    squarechart as squarec,  # type: ignore
    squarechart as squarech,  # type: ignore
    squarechart as squarecha,  # type: ignore
    squarechart as squarechar,  # type: ignore
    squarechart as squarechart,  # type: ignore
    textchart as text,  # type: ignore
    textchart as textc,  # type: ignore
    textchart as textch,  # type: ignore
    textchart as textcha,  # type: ignore
    textchart as textchar,  # type: ignore
    textchart as textchart,  # type: ignore
    tickchart as tick,  # type: ignore
    tickchart as tickc,  # type: ignore
    tickchart as tickch,  # type: ignore
    tickchart as tickcha,  # type: ignore
    tickchart as tickchar,  # type: ignore
    tickchart as tickchart,  # type: ignore
    trailchart as trail,  # type: ignore
    trailchart as trailc,  # type: ignore
    trailchart as trailch,  # type: ignore
    trailchart as trailcha,  # type: ignore
    trailchart as trailchar,  # type: ignore
    trailchart as trailchart,  # type: ignore
    boxplotchart as boxplot,  # type: ignore
    boxplotchart as boxplotc,  # type: ignore
    boxplotchart as boxplotch,  # type: ignore
    boxplotchart as boxplotcha,  # type: ignore
    boxplotchart as boxplotchar,  # type: ignore
    boxplotchart as boxplotchart,  # type: ignore
    errorbandchart as errorband,  # type: ignore
    errorbandchart as errorbandc,  # type: ignore
    errorbandchart as errorbandch,  # type: ignore
    errorbandchart as errorbandcha,  # type: ignore
    errorbandchart as errorbandchar,  # type: ignore
    errorbandchart as errorbandchart,  # type: ignore
    errorbarchart as errorbar,  # type: ignore
    errorbarchart as errorbarc,  # type: ignore
    errorbarchart as errorbarch,  # type: ignore
    errorbarchart as errorbarcha,  # type: ignore
    errorbarchart as errorbarchar,  # type: ignore
    errorbarchart as errorbarchart,  # type: ignore
)

from ._altair_mapper import (
    arcchart_ as arc_,  # type: ignore
    arcchart_ as arcc_,  # type: ignore
    arcchart_ as arcch_,  # type: ignore
    arcchart_ as arccha_,  # type: ignore
    arcchart_ as arcchar_,  # type: ignore
    arcchart_ as arcchart_,  # type: ignore
    areachart_ as area_,  # type: ignore
    areachart_ as areac_,  # type: ignore
    areachart_ as areach_,  # type: ignore
    areachart_ as areacha_,  # type: ignore
    areachart_ as areachar_,  # type: ignore
    areachart_ as areachart_,  # type: ignore
    barchart_ as bar_,  # type: ignore
    barchart_ as barc_,  # type: ignore
    barchart_ as barch_,  # type: ignore
    barchart_ as barcha_,  # type: ignore
    barchart_ as barchar_,  # type: ignore
    barchart_ as barchart_,  # type: ignore
    circlechart_ as circle_,  # type: ignore
    circlechart_ as circlec_,  # type: ignore
    circlechart_ as circlech_,  # type: ignore
    circlechart_ as circlecha_,  # type: ignore
    circlechart_ as circlechar_,  # type: ignore
    circlechart_ as circlechart_,  # type: ignore
    geoshapechart_ as geoshape_,  # type: ignore
    geoshapechart_ as geoshapec_,  # type: ignore
    geoshapechart_ as geoshapech_,  # type: ignore
    geoshapechart_ as geoshapecha_,  # type: ignore
    geoshapechart_ as geoshapechar_,  # type: ignore
    geoshapechart_ as geoshapechart_,  # type: ignore
    imagechart_ as image_,  # type: ignore
    imagechart_ as imagec_,  # type: ignore
    imagechart_ as imagech_,  # type: ignore
    imagechart_ as imagecha_,  # type: ignore
    imagechart_ as imagechar_,  # type: ignore
    imagechart_ as imagechart_,  # type: ignore
    linechart_ as line_,  # type: ignore
    linechart_ as linec_,  # type: ignore
    linechart_ as linech_,  # type: ignore
    linechart_ as linecha_,  # type: ignore
    linechart_ as linechar_,  # type: ignore
    linechart_ as linechart_,  # type: ignore
    pointchart_ as point_,  # type: ignore
    pointchart_ as pointc_,  # type: ignore
    pointchart_ as pointch_,  # type: ignore
    pointchart_ as pointcha_,  # type: ignore
    pointchart_ as pointchar_,  # type: ignore
    pointchart_ as pointchart_,  # type: ignore
    rectchart_ as rect_,  # type: ignore
    rectchart_ as rectc_,  # type: ignore
    rectchart_ as rectch_,  # type: ignore
    rectchart_ as rectcha_,  # type: ignore
    rectchart_ as rectchar_,  # type: ignore
    rectchart_ as rectchart_,  # type: ignore
    rulechart_ as rule_,  # type: ignore
    rulechart_ as rulec_,  # type: ignore
    rulechart_ as rulech_,  # type: ignore
    rulechart_ as rulecha_,  # type: ignore
    rulechart_ as rulechar_,  # type: ignore
    rulechart_ as rulechart_,  # type: ignore
    squarechart_ as square_,  # type: ignore
    squarechart_ as squarec_,  # type: ignore
    squarechart_ as squarech_,  # type: ignore
    squarechart_ as squarecha_,  # type: ignore
    squarechart_ as squarechar_,  # type: ignore
    squarechart_ as squarechart_,  # type: ignore
    textchart_ as text_,  # type: ignore
    textchart_ as textc_,  # type: ignore
    textchart_ as textch_,  # type: ignore
    textchart_ as textcha_,  # type: ignore
    textchart_ as textchar_,  # type: ignore
    textchart_ as textchart_,  # type: ignore
    tickchart_ as tick_,  # type: ignore
    tickchart_ as tickc_,  # type: ignore
    tickchart_ as tickch_,  # type: ignore
    tickchart_ as tickcha_,  # type: ignore
    tickchart_ as tickchar_,  # type: ignore
    tickchart_ as tickchart_,  # type: ignore
    trailchart_ as trail_,  # type: ignore
    trailchart_ as trailc_,  # type: ignore
    trailchart_ as trailch_,  # type: ignore
    trailchart_ as trailcha_,  # type: ignore
    trailchart_ as trailchar_,  # type: ignore
    trailchart_ as trailchart_,  # type: ignore
    boxplotchart_ as boxplot_,  # type: ignore
    boxplotchart_ as boxplotc_,  # type: ignore
    boxplotchart_ as boxplotch_,  # type: ignore
    boxplotchart_ as boxplotcha_,  # type: ignore
    boxplotchart_ as boxplotchar_,  # type: ignore
    boxplotchart_ as boxplotchart_,  # type: ignore
    errorbandchart_ as errorband_,  # type: ignore
    errorbandchart_ as errorbandc_,  # type: ignore
    errorbandchart_ as errorbandch_,  # type: ignore
    errorbandchart_ as errorbandcha_,  # type: ignore
    errorbandchart_ as errorbandchar_,  # type: ignore
    errorbandchart_ as errorbandchart_,  # type: ignore
    errorbarchart_ as errorbar_,  # type: ignore
    errorbarchart_ as errorbarc_,  # type: ignore
    errorbarchart_ as errorbarch_,  # type: ignore
    errorbarchart_ as errorbarcha_,  # type: ignore
    errorbarchart_ as errorbarchar_,  # type: ignore
    errorbarchart_ as errorbarchart_,  # type: ignore
)

from .shortcuts.scatter import scatter as sc
from .shortcuts.scatter import scatter as sca
from .shortcuts.scatter import scatter as scat
from .shortcuts.scatter import scatter as scatt
from .shortcuts.scatter import scatter as scatte
from .shortcuts.scatter import scatter as scatter
from .shortcuts.histogram import histogram as hist
from .shortcuts.histogram import histogram as histo
from .shortcuts.histogram import histogram as histog
from .shortcuts.histogram import histogram as histogr
from .shortcuts.histogram import histogram as histogra
from .shortcuts.histogram import histogram as histogram

# Experimental commands might not have dependencies specified in setup.py
try:
    from .experimental.flan_t5_base import flan_t5_base
    from .experimental.generate_split import generate_split
    from .experimental.nnlinear import nnlinear
    from .experimental.bard import bard
    from .experimental.chatgpt import chatgpt
    from .experimental.streamlit import streamlit
    from .experimental.scatter_seaborn import scatter as scatter_seaborn
except:
    pass

from .nn.gpt2 import gpt2
from .export import export

# from .nn.finetune import finetune
from .fine_tuning.fthuggingface import fthuggingface
from .nn.pipeline import pipeline

from .data.thegraph.utils import (
    extract_api_requests,
    tothegraph,
    searchframes,
    searchqueries,
)

from .data.thegraph.uniswap_v3.pool_page import *

from .doedit import run_active_python_script

# from IPython.core.getipython import get_ipython

# get_ipython().run_line_magic("shortcut", "Ctrl-D", "run_active_python_script")
# from time import time

# t0 = time()
# try:
#     import keyboard

# keyboard.add_hotkey("ctrl+f9", run_active_python_script)
# except:
#     pass  # fails in github actions
# print(time() - t0)

from .browse import browse_off

# keyboard.as

# import keyboard

# keyboard.add_hotkey("ctrl+f9", lambda: input("How are you?"))
try:
    from ._hotkeys import hotkey_ctrl_f9

    hotkey_ctrl_f9(run_active_python_script)
except:
    pass


# from pynput import keyboard

# listener = keyboard.GlobalHotKeys({"<ctrl>+h": run_active_python_script})
# listener.start()

from .fine_tuning.ftgpt import ftgpt, askgpt
from .fine_tuning.fthuggingface import fthuggingface, askhuggingface
from .predict import predict
from .sample import sample

import warnings

warnings.filterwarnings("ignore")

pdexplorer/__init__.py
pandichef/pdexplorer
import webbrowser
import altair as alt
from ._dataset import current
from ._commandarg import parse, parse_options
from ._search import search_iterable

# monkey patch callable method
def _call(self):
    filename = "tmpqbxcec2k.html"
    self.save(filename)
    from ._webbrowser import webbrowser_open

    webbrowser_open(filename)


def _transform_calculate_variable_labels(self):
    reversed_variable_labels = {
        v: "datum." + k for k, v in current.metadata["variable_labels"].items()
    }
    return alt.Chart.transform_calculate(self, **reversed_variable_labels)


alt.Chart.__call__ = _call  # type: ignore
alt.Chart.transform_calculate_variable_labels = _transform_calculate_variable_labels  # type: ignore
alt.LayerChart.__call__ = _call  # type: ignore
alt.LayerChart.transform_calculate_variable_labels = _transform_calculate_variable_labels  # type: ignore
alt.HConcatChart.__call__ = _call  # type: ignore
alt.HConcatChart.transform_calculate_variable_labels = _transform_calculate_variable_labels  # type: ignore
alt.VConcatChart.__call__ = _call  # type: ignore
alt.VConcatChart.transform_calculate_variable_labels = _transform_calculate_variable_labels  # type: ignore


# def _get_kwargs_list(commandarg, yX=True):
def _get_kwargs_list(commandarg, yX=True, use_labels=True):
    """get encodings as keywords"""
    _ = parse(commandarg)
    non_x_y_encodings = parse_options(_["options"], split_values=True)
    for k, v in non_x_y_encodings.items():
        if len(v) == 1:
            non_x_y_encodings[k] = v[0]

    varlist = search_iterable(current.df.columns, _["anything"])
    if yX:
        if len(varlist) == 1:
            yvar = varlist[0]
            xvars = list(current.df.columns)
            xvars.remove(yvar)
        else:
            yvar = varlist.pop(0)
            xvars = varlist
        kwargs_list = []
        for xvar in xvars:
            if use_labels:
                this_kwargs = {
                    # "y": current.metadata["variable_labels"][yvar],
                    # "x": current.metadata["variable_labels"][xvar],
                    "y": alt.Y(yvar, title=current.metadata["variable_labels"][yvar]),
                    "x": alt.X(xvar, title=current.metadata["variable_labels"][xvar]),
                }
            else:
                this_kwargs = {"y": yvar, "x": xvar}
            this_kwargs.update(non_x_y_encodings)
            kwargs_list.append(this_kwargs)
        return kwargs_list
    else:
        if len(varlist) == 1:
            xvar = varlist[0]
            yvars = list(current.df.columns)
            yvars.remove(xvar)
        else:
            xvar = varlist.pop()
            yvars = varlist
        kwargs_list = []
        for yvar in yvars:
            # this_kwargs = {"y": yvar, "x": xvar}
            if use_labels:
                this_kwargs = {
                    # "y": current.metadata["variable_labels"][yvar],
                    # "x": current.metadata["variable_labels"][xvar],
                    "y": alt.Y(yvar, title=current.metadata["variable_labels"][yvar]),
                    "x": alt.X(xvar, title=current.metadata["variable_labels"][xvar]),
                }
            else:
                this_kwargs = {"y": yvar, "x": xvar}
            this_kwargs.update(non_x_y_encodings)
            kwargs_list.append(this_kwargs)
        return kwargs_list


def _chart(
    mark_method,
    commandarg=None,
    use_labels=True,
    yX=False,
    stacked=False,
    *args,
    **kwargs
):
    """
    yX: If True, the "v1 v2 v3" is mapped as "y x1 x2"
    stacked: If False, all charts are layered onto a single grid
    """
    _kwargs_list = (
        _get_kwargs_list(commandarg, yX=yX, use_labels=use_labels) if commandarg else []
    )
    # print(_kwargs_list)
    if len(_kwargs_list) == 0:
        return mark_method(*args, **kwargs)
    if len(_kwargs_list) == 1:
        return mark_method(*args, **kwargs).encode(**_kwargs_list[0])
    else:
        if not stacked:
            # sugar for layered charts in the Stata-like manner
            _ = parse(commandarg)
            layered_encodings = parse_options(_["options"], split_values=True)
            for k, v in layered_encodings.items():
                # Becuase Altair doesn't like lists of length 1
                if len(v) == 1:
                    layered_encodings[k] = v[0]
            if yX:
                xvars = _["anything"].split()
                yvar = xvars.pop(0)
                layered_encodings.update({"color": "key:N"})
                layered_encodings.update({"x": "value:Q"})
                layered_encodings.update(
                    {
                        "y": alt.Y(
                            yvar, title=current.metadata["variable_labels"][yvar]
                        )
                        if use_labels
                        else yvar
                    }
                )
                if use_labels:
                    # .transform_calculate_variable_labels()
                    return (
                        mark_method(*args, **kwargs)
                        .encode(**layered_encodings)
                        .transform_fold(
                            list(
                                map(
                                    lambda x: current.metadata["variable_labels"][x],
                                    xvars,
                                )
                            )
                        )
                    )
                else:
                    return (
                        mark_method(*args, **kwargs)
                        .encode(**layered_encodings)
                        .transform_fold(xvars)
                    )
            else:
                yvars = _["anything"].split()
                xvar = yvars.pop()
                layered_encodings.update({"color": "key:N"})
                layered_encodings.update({"y": "value:Q"})
                layered_encodings.update(
                    {
                        "x": alt.X(
                            xvar, title=current.metadata["variable_labels"][xvar]
                        )
                        if use_labels
                        else xvar
                    }
                )
                # print(layered_encodings)
                # print(yvars)
                if use_labels:
                    return (
                        mark_method(*args, **kwargs)
                        .transform_calculate_variable_labels()
                        .encode(**layered_encodings)
                        .transform_fold(
                            list(
                                map(
                                    lambda x: current.metadata["variable_labels"][x],
                                    yvars,
                                )
                            )
                        )
                    )
                else:
                    return (
                        mark_method(*args, **kwargs)
                        .encode(**layered_encodings)
                        .transform_fold(yvars)
                    )
        else:
            # Stacked version here
            vconcat = mark_method(*args, **kwargs).encode(**_kwargs_list[0])
            for i, _kwargs in enumerate(_kwargs_list):
                if i > 0:
                    vconcat = vconcat & (
                        mark_method(*args, **kwargs).encode(**_kwargs_list[i])
                    )
            return vconcat


marktypes = [  # https://altair-viz.github.io/user_guide/marks/index.html
    "arc",  # https://altair-viz.github.io/user_guide/marks/arc.html#user-guide-arc-marks
    "area",  # https://altair-viz.github.io/user_guide/marks/area.html#user-guide-area-marks
    "bar",  # https://altair-viz.github.io/user_guide/marks/bar.html#user-guide-bar-marks
    "circle",  # https://altair-viz.github.io/user_guide/marks/circle.html#user-guide-circle-marks
    "geoshape",  # https://altair-viz.github.io/user_guide/marks/geoshape.html#user-guide-geoshape-marks
    "image",  # https://altair-viz.github.io/user_guide/marks/image.html#user-guide-image-marks
    "line",  # https://altair-viz.github.io/user_guide/marks/line.html#user-guide-line-marks
    "point",  # https://altair-viz.github.io/user_guide/marks/point.html#user-guide-point-marks
    "rect",  # https://altair-viz.github.io/user_guide/marks/rect.html#user-guide-rect-marks
    "rule",  # https://altair-viz.github.io/user_guide/marks/rule.html#user-guide-rule-marks
    "square",  # https://altair-viz.github.io/user_guide/marks/square.html#user-guide-square-marks
    "text",  # https://altair-viz.github.io/user_guide/marks/text.html#user-guide-text-marks
    "tick",  # https://altair-viz.github.io/user_guide/marks/tick.html#user-guide-tick-marks
    "trail",  # https://altair-viz.github.io/user_guide/marks/trail.html#user-guide-trail-marks
    "boxplot",  # https://altair-viz.github.io/user_guide/marks/boxplot.html#user-guide-boxplot-marks
    "errorband",  # https://altair-viz.github.io/user_guide/marks/errorband.html#user-guide-errorband-marks
    "errorbar",  # https://altair-viz.github.io/user_guide/marks/errorbar.html#user-guide-errorbar-marks
]

fnc = """def {marktype}chart(commandarg=None, use_labels=True, *args, **kwargs):
    mark_method = alt.Chart(current.df).mark_{marktype}
    _chart(mark_method, commandarg, use_labels, *args, **kwargs)()
"""

fnc2 = """def {marktype}chart_(commandarg=None, use_labels=True, calculate_variable_labels=True, *args, **kwargs):
    mark_method = alt.Chart(current.df).mark_{marktype}
    if calculate_variable_labels:
        return _chart(mark_method, commandarg, use_labels, *args, **kwargs).transform_calculate_variable_labels()
    else:
        return _chart(mark_method, commandarg, use_labels, *args, **kwargs)
"""
# fnc = """def {marktype}chart(use_labels=True, *args, **kwargs):
#     if use_labels:
#         mark_method = alt.Chart(current.df_labeled).mark_{marktype}
#     else:
#         mark_method = alt.Chart(current.df).mark_{marktype}
#     return _chart(mark_method, use_labels=use_labels, *args, **kwargs)
# """

for marktype in marktypes:
    exec(fnc.format(marktype=marktype))
    exec(fnc2.format(marktype=marktype))

##################################################################
# Stata-like Sugar Below
# def histogram(varname):
#     barchart().encode(alt.X(varname, bin=True), y="count()")()  # type: ignore


# def scatter(commandarg):
#     circlechart(commandarg)()  # type: ignore

pdexplorer/_altair_mapper.py
pandichef/pdexplorer
from ._dataset import current
from ._quietly import quietly
from copy import copy
from ._print_horizontal_line import print_horizontal_line
from .use import use
import functools
from .sort import sort

# decorator that indicates that a given command supports the by context
def byable(command):
    @functools.wraps(command)
    def wrapper(*args, **kwargs):
        global current
        if current.byvar:
            orig_current = copy(current)
            try:
                with quietly():
                    sort(current.byvar)
                # Loop through byvar
                for this in current.df[current.byvar].unique().tolist():
                    _df = orig_current.df
                    _df = _df[_df[current.byvar] == this]  # filtered
                    with quietly():
                        use(_df)
                    print_horizontal_line()
                    print(f"-> {current.byvar} = {this}")
                    command(*args, **kwargs)  # by context cannot support return values
                current = orig_current
            except:
                current = orig_current
        else:
            return command(*args, **kwargs)

    return wrapper


# by context manager
class by:
    def __init__(self, byvar: str) -> None:
        current.byvar = byvar

    def __enter__(self):
        return None  # The part after "as"

    def __exit__(self, exc_type, exc_value, traceback_obj):
        current.byvar = None

pdexplorer/_by.py
pandichef/pdexplorer
import re
from ._dataset import current
from ._search import search_iterable


def _clean(string_with_extra_spaces: str) -> str:
    cleaned = " ".join(string_with_extra_spaces.split())
    return cleaned


def check_syntax(_: dict, syntax: str) -> dict:
    # _ is the parsed commandarg
    updates = {}
    if syntax is not None and syntax != "" and syntax.find("varlist") > -1:
        expanded_variable_names = search_iterable(current.df.columns, _["anything"])
        for name in expanded_variable_names:
            assert (
                name in current.df.columns
            ), f"Syntax specifies a varlist, but {name} not found in current dataset."
        updates.update({"varlist": " ".join(expanded_variable_names)})
    else:
        pass
    return updates


def parse_options(options_string: str, split_values=False) -> dict:
    # split_values is used in _altair_mapper
    if options_string is None:
        return {}
    options_string = options_string.replace("( ", "(")
    options_string = options_string.replace(" )", ")")

    result = {}
    pattern = r"(\w+)\(([^)]*)\)"
    matches = re.findall(pattern, options_string)

    for key, value in matches:
        result[key] = value.strip()

    remaining_words = re.sub(pattern, "", options_string).split()
    for word in remaining_words:
        result[word] = True

    if split_values:
        return {key: value.split(" ") for key, value in result.items()}
    else:
        return result


def parse(commandarg: str, syntax: str = "") -> dict:
    commandarg = commandarg.replace("==", "__eq__").replace("!=", "__neq__")
    if (
        commandarg.startswith("if ")
        or commandarg.startswith("in ")
        or commandarg.startswith("using ")
    ):
        commandarg = " " + commandarg
    split_identifiers = (
        " in ",
        " if ",
        " using ",
        ",",
        " [",
        "]",
        "=",  # check for = before weights only
    )
    split_locations = {}
    end_of_anything = None
    for i, split_identifier in enumerate(split_identifiers):
        # this_location = commandarg.find(split_identifier)
        split_locations[split_identifier] = commandarg.find(split_identifier)
        # print(split_identifier)
        # print(split_locations[split_identifier])
        if (  # exclude when "=" is after the open bracket
            split_identifier == "="
            and split_locations[" ["] > -1
            and split_locations[split_identifier] > split_locations[" ["]
        ):
            # print(split_locations[split_identifier])
            # print(split_locations[" ["])
            split_locations[split_identifier] = -1
        if split_locations[split_identifier] > -1:
            if end_of_anything:
                end_of_anything = min(
                    split_locations[split_identifier], end_of_anything
                )
            else:
                end_of_anything = split_locations[split_identifier]
    # print(split_locations)
    sorted_splits = sorted(
        split_locations.items(), key=lambda x: x[1]
    )  # list of tuples
    _ = {}  # parsed commandarg is represented as _
    for index, (identifier, position) in enumerate(sorted_splits):
        if position == -1:
            _[identifier] = None
        else:
            if identifier == "[":
                startpos = position + len(identifier)
                endpos = split_locations["]"]
                _[identifier] = _clean(commandarg[startpos:endpos])
            elif identifier != "]":
                startpos = position + len(identifier)  # + 1
                try:
                    endpos = sorted_splits[index + 1][1]
                    _[identifier] = _clean(commandarg[startpos:endpos])
                except IndexError:
                    _[identifier] = _clean(commandarg[startpos:])
        # if parsed[identifier] and :  # i.e., not None
        #     parsed[identifier] = " ".join(parsed[identifier].split())

    _["anything"] = _clean(commandarg[:end_of_anything])
    if _["anything"] == "":
        _["anything"] = None
    if _[" if "]:
        _[" if "] = _[" if "].replace("__eq__", "==").replace("__neq__", "!=")

    # rename
    _["weight"] = _[" ["]
    del _[" ["]
    if _["weight"] is None:
        del _["]"]  # hack
    _["options"] = _[","]
    del _[","]
    _["if"] = _[" if "]
    del _[" if "]
    _["in"] = _[" in "]
    del _[" in "]
    _["using"] = _[" using "]
    del _[" using "]
    # parsed["exp"] = parsed["="]

    # See https://www.stata.com/manuals13/psyntax.pdf
    parsed_options = parse_options(_["options"])
    _.update(parsed_options)
    # _["options"] = parse_options(_["options"], split_values=True)
    updates = check_syntax(_, syntax)  # raise error is syntax is invalid
    _.update(updates)  # e.g., varlist added if passes validation
    return _


def parse_if_condition(if_condition_string: str) -> dict:
    return {}  # todo


pdexplorer/_commandarg.py
pandichef/pdexplorer
import io
import pandas as pd
from copy import copy
from typing import Optional, Callable
from ._search import search_iterable


# import dtale
# from dtale.views import DtaleData

# df = pd.DataFrame()
# class WithinByContextManager(Exception):
#     pass


# varlist_as_list = varlist.split()
# yvar = varlist_as_list[0]
# xvars = varlist_as_list[1:]
# xvars = search_iterable(current.df.columns, " ".join(xvars))
# x_train = current.df.dropna()[xvars].values
# y_train = current.df.dropna()[[yvar]].values
# X = torch.tensor(x_train).to(torch.float32)
# y = torch.tensor(y_train).to(torch.float32)


class Dataset:
    METADATA_DEFAULT = {"data_label": "", "variable_labels": {}}

    def __init__(self) -> None:
        # self.dtale_browser: DtaleData | None = None
        self._clear_data()
        self._clear_settings()
        self._clear_results()
        # context manager variables #
        self.byvar: str | None = None
        self.quietly = False

    def _clear_data(self, include_preserved=True):
        self._df: pd.DataFrame = pd.DataFrame()
        self.metadata = copy(Dataset.METADATA_DEFAULT)
        if include_preserved:
            self._df_preserved: pd.DataFrame = pd.DataFrame()
            self.metadata_preserved = copy(Dataset.METADATA_DEFAULT)
            self.has_preserved = False

    def _clear_results(self):
        self.stored_results = {
            "r": {},  # results from general commands
            "e": {},  # results from estimation commands
            "s": {},  # results from parsing commands
            "n": {},  # commands that do not store in r(), e(), or s()
            "c": {},  # contains the values of system parameters and settings
        }
        self.predict_fnc: Optional[Callable[[str], None]] = None
        self.methods: dict = {}  # See statsmodel old documentation
        self.properties: dict = {}  # See statsmodel old documentation

    def _clear_settings(self):
        # self.quietly = False
        self.dtale_browser = None  # type DtaleData
        self.has_preserved = False
        self.active_python_script: Optional[str] = None  # see doedit
        self.last_openai_ftjob_id: Optional[str] = None
        self.last_huggingface_ftmodel_dir: Optional[str] = None
        self.browse_turned_on = False
        self.use_torch_bfloat16 = True
        # self.last_huggingface_tokenizer = None
        self.captured_output = io.StringIO()

    # @property
    def clear(self):
        self._clear_data(include_preserved=False)

    # @property
    def clearall(self):
        self._clear_data()
        self._clear_settings()
        self._clear_results()

    @property
    def df(self):
        # returns a copy of df
        # use current._df to return a reference
        # print(
        #     "Making a copy of current dataframe.  Use current._df to get a reference to the current dataframe."
        # )
        # return self._df.copy()
        return self._df

    @property
    def df_labeled(self):
        return self._df.rename(columns=self.metadata["variable_labels"])

    @df.setter
    def df(self, value: pd.DataFrame):
        # if self.metadata["variable_labels"]:
        #     self._df = value
        # else:
        column_mapping = {}
        for col in value.columns:
            if isinstance(col, str):  # type error without this
                column_mapping[col] = (
                    col.replace(" ", "")
                    .replace("(", "")
                    .replace(")", "")
                    .replace(",", "")
                    .lower()
                )  # Stata default is to remove spaces
            else:
                column_mapping[col] = col
        self._df = value.rename(columns=column_mapping)
        self.metadata["variable_labels"] = {
            value: key for key, value in column_mapping.items()
        }

    @property
    def df_preserved(self):
        return self._df_preserved

    @df_preserved.setter
    def df_preserved(self, value):
        self._df_preserved = value
        self.has_preserved = True

    def split(self, name: str) -> pd.DataFrame:
        _return_df = self._df.query(f"split == '{name}'", inplace=False)
        if _return_df.empty:
            print(f'Split "{name}" not found.  Split values:')
            print(self._df.split.value_counts())
        return _return_df

    def get_huggingface_dataset(
        self, varlist: Optional[str] = None, split: Optional[str] = None
    ):
        import datasets

        if not varlist:
            varlist = "*"
        varlist = " ".join(search_iterable(self._df.columns, varlist))
        if split:
            dataset = datasets.Dataset.from_pandas(
                self._df[varlist.split()].query(f"""split=='{split}'""")
            )
        else:
            dataset = datasets.Dataset.from_pandas(self._df[varlist.split()])
        return dataset

    def get_pytorch_dataset(
        self, varlist: Optional[str] = None, split: Optional[str] = None, tokenizer=None
    ):

        # from transformers import AutoTokenizer

        if not varlist:
            varlist = "*"
        varlist = " ".join(search_iterable(self._df.columns, varlist))
        xvars = varlist.split()
        yvar = xvars.pop(0)
        df_split = self._df.copy()

        if split:
            df_split = df_split.query(f"""split=='{split}'""")

        # ds = self.get_huggingface_dataset(varlist, split)
        # model expects the argument to be named labels #
        # ds = ds.rename_column(yvar, "labels")
        # ds.set_format("torch")
        # model_name = "distilbert-base-uncased"  # bert-base-cased is much larger #
        # tokenizer = AutoTokenizer.from_pretrained(model_name)

        # def tokenize_function(examples):
        #     return tokenizer(examples[xvar], padding="max_length", truncation=True)

        # tokenized_datasets = dataset.map(tokenize_function, batched=True,)

        # dataset.set_format(type="torch")
        # https://huggingface.co/docs/datasets/use_with_pytorch#dataset-format #
        # device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        # ds = dataset.with_format("torch", device=device)
        # return ds

        # from torch.utils.data import DataLoader
        import torch
        from torch.utils.data import Dataset as PyTorchDataset

        class _PyTorchDataset(PyTorchDataset):
            # PyTorch dataset is a iterable that separates the X variables from the y variable
            def __init__(self, df):

                # self.nobs = len(df)
                # varlist_as_list = varlist.split()
                # yvar = varlist_as_list[0]
                # xvars = varlist_as_list[1:]
                # xvars = search_iterable(df.columns, " ".join(xvars))

                x_train = df.dropna()[xvars].values
                y_train = df.dropna()[[yvar]].values
                # below fails if not float32
                self.x_train = torch.tensor(x_train).to(torch.float32)
                self.y_train = torch.tensor(y_train).to(torch.float32)

                # x_df = df[xvars]
                # y_series = df[yvar]

                # self.x_train = torch.tensor(x_df.values)
                # self.y_train = torch.tensor(y_series)

            def __len__(self):
                return len(self.y_train)

            def __getitem__(self, idx):
                # idx = torch.long(idx)
                # idx = torch.arange(self.nobs, dtype=torch.long)
                # print(type(idx))
                return self.x_train[idx], self.y_train[idx]

        return _PyTorchDataset(df_split)

    def get_pytorch_dataloader(
        self,
        varlist: Optional[str] = None,
        split: Optional[str] = None,
        batch_size=100,
        shuffle=False,
    ):
        from torch.utils.data import DataLoader

        # varlist = " ".join(search_iterable(self._df.columns, varlist))
        ds = self.get_pytorch_dataset(varlist, split)
        dl = DataLoader(dataset=ds, batch_size=batch_size, shuffle=shuffle)  # type: ignore
        return dl


current = Dataset()

# def clearall_global():
#     global current
#     current = Dataset()


# current = Dataset()  # https://www.stata.com/stata16/multiple-datasets-in-memory/
# current = default  # https://www.stata.com/stata16/multiple-datasets-in-memory/
# df = current._df
# print(current._df)
# print(df)
# def _supports_byvar():
#     if current.within_by_context_manager:
#         raise WithinByContextManager

pdexplorer/_dataset.py
pandichef/pdexplorer
def _get_custom_attributes(self):
    # Create a dictionary to store non-method attributes and their values
    custom_properties = {}
    custom_methods = {}

    # Iterate through the attributes using dir()
    for attr_name in dir(self):
        attr_value = getattr(self, attr_name)

        # Exclude attributes that are callable (methods)
        # print(type(attr_value))
        if not attr_name.startswith("_"):
            if callable(attr_value):
                # Add the attribute and its value to the dictionary
                docstring = attr_value.__doc__
                if docstring:
                    first_line = attr_value.__doc__.split("\n")[1].strip()
                else:
                    first_line = ""
                custom_methods[attr_name] = first_line
            else:
                custom_properties[attr_name] = attr_value

    return custom_methods, custom_properties

pdexplorer/_get_custom_attributes.py
pandichef/pdexplorer
# https://pynput.readthedocs.io/en/latest/keyboard.html#monitoring-the-keyboard #
# https://github.com/moses-palmer/pynput/issues/20#issuecomment-290649632 #


def hotkey_ctrl_f9(fnc):
    """
    compare to these two alternatives:
    1. import keyboard; keyboard.add_hotkey("ctrl+f9", lambda: print("hello"))
    2. import pynput; pynput.keyboard.GlobalHotKeys({"<ctrl>+<f9>": lambda: print("hello")})
    * all methods freeze ipython when input() is used a python command
    * all methods causes issues in ubuntu
    * pynput appear to be faster than the keyboard package, so I used it vs keyboard
    """
    # from pynput import keyboard

    # LEFT_COMBINATION = {keyboard.Key.ctrl_l, keyboard.Key.f9}
    # RIGHT_COMBINATION = {keyboard.Key.ctrl_r, keyboard.Key.f9}

    # key_set = set()

    # def on_press(key):
    #     if key in LEFT_COMBINATION.union(RIGHT_COMBINATION):
    #         key_set.add(key)
    #         if all(k in key_set for k in LEFT_COMBINATION) or all(
    #             k in key_set for k in RIGHT_COMBINATION
    #         ):
    #             fnc()

    # def on_release(key):
    #     try:
    #         key_set.remove(key)
    #     except KeyError:
    #         pass

    # # ...or, in a non-blocking fashion:
    # listener = keyboard.Listener(on_press=on_press, on_release=on_release)
    # listener.start()
    from pynput import keyboard

    LEFT_COMBINATION = {keyboard.Key.ctrl_l, keyboard.Key.f9}
    RIGHT_COMBINATION = {keyboard.Key.ctrl_r, keyboard.Key.f9}

    key_set = set()

    def on_press(key):
        if key in LEFT_COMBINATION.union(RIGHT_COMBINATION):
            key_set.add(key)

            if LEFT_COMBINATION.issubset(key_set) or RIGHT_COMBINATION.issubset(
                key_set
            ):
                fnc()
                key_set.clear()  # Clear the set after calling fnc()

    def on_release(key):
        key_set.discard(
            key
        )  # Remove the key without raising an exception if it doesn't exist

    listener = keyboard.Listener(on_press=on_press, on_release=on_release)
    listener.start()

pdexplorer/_hotkeys.py
pandichef/pdexplorer
from ._dataset import current
from ._search import search_iterable


def _patsify(varlist: str) -> str:
    df = current.df
    patsy_formula = " + ".join(search_iterable(df.columns, varlist))
    patsy_formula = patsy_formula.replace(" + ", " ~ ", 1)
    return patsy_formula

pdexplorer/_patsify.py
pandichef/pdexplorer
from io import StringIO
import sys
from ._dataset import current


def _print(obj):
    """Modified print statement using global settings"""
    if not current.quietly and not current.browse_turned_on:
        print(obj)
    else:
        # new_captured_output = StringIO()
        # current.captured_output = new_captured_output
        current.captured_output.write(str(obj) + "\n")


# def _print(obj):
#     """Modified print statement using global settings"""
#     if not current.quietly and not current.browse_turned_on:
#         print(obj)
#     else:
#         original_stdout = sys.stdout
#         captured_output = StringIO()
#         sys.stdout = captured_output
#         print(obj)
#         sys.stdout = original_stdout
#         return captured_output.getvalue()

pdexplorer/_print.py
pandichef/pdexplorer
import shutil


def print_horizontal_line():
    # Get the terminal width
    terminal_width, _ = shutil.get_terminal_size()

    # Create a horizontal line
    line = "-" * terminal_width

    # Print the horizontal line
    print(line)

pdexplorer/_print_horizontal_line.py
pandichef/pdexplorer
# "profile" isn't a stata command
# these commands are used for the "cf" command
import os
from ._dataset import current
import webbrowser
import tempfile
from .use import _use
from .keep import _keep
from ._search import search_iterable


def sweetviz_profile(varlist=None, compare_to=None):
    import sweetviz as sv

    if varlist:
        columns_to_keep = search_iterable(current.df.columns, varlist)
    else:
        columns_to_keep = list(current.df.columns)

    if compare_to is not None:
        # https://www.stata.com/manuals/dcf.pdf
        master_df = _keep(current.df, columns_to_keep)
        using_df = _keep(_use(compare_to)[0], columns_to_keep)
        report = sv.compare(master_df, using_df)
    else:
        report = sv.analyze(current.df)
    report.show_html()


def ydata_profile(varlist=None, compare_to=None):
    # This primarily used for the cf command since dtale doesn't apparently
    # support comparisons directly
    # Note: this isn't a Stata command, but it should have been
    from ydata_profiling import ProfileReport

    if varlist:
        columns_to_keep = search_iterable(current.df.columns, varlist)
    else:
        columns_to_keep = list(current.df.columns)

    if compare_to is not None:
        # https://ydata-profiling.ydata.ai/docs/master/pages/use_cases/comparing_datasets.html
        # https://www.stata.com/manuals/dcf.pdf
        # using_df = _use(compare_to)[0]
        master_df = _keep(current.df, columns_to_keep)
        using_df = _keep(_use(compare_to)[0], columns_to_keep)
        master_report = ProfileReport(master_df, title="master")
        using_report = ProfileReport(using_df, title="using")
        report = master_report.compare(using_report)
    else:
        # see  https://ydata-profiling.ydata.ai/docs/master/pages/use_cases/metadata.html
        master_df = _keep(current.df, columns_to_keep)
        report = master_df.profile_report(
            title=current.metadata["data_label"].split("\n")[0],
            dataset={"description": current.metadata["data_label"]},
            variables={"descriptions": current.metadata["variable_labels"]},
        )

    temp_dir = tempfile.gettempdir()
    temp_file_name = os.path.join(temp_dir, "tmpqbxcec2k.html")
    report.to_file(temp_file_name)
    webbrowser.open(temp_file_name)

pdexplorer/_profile.py
pandichef/pdexplorer
import sys
from io import StringIO
import contextlib
from ._dataset import current
from functools import wraps


@contextlib.contextmanager
def quietly(hard=False):
    if hard:
        original_stdout = sys.stdout
        captured_output_hard = StringIO()
        sys.stdout = captured_output_hard
        yield captured_output_hard
        sys.stdout = original_stdout
        current.captured_output = captured_output_hard
    else:
        original_value = current.quietly
        current.quietly = True
        current.captured_output = StringIO()
        # try:
        yield current.captured_output
        # finally:
        current.quietly = original_value


def quietly_decorator(hard=False):
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            with quietly(hard=hard) as captured_output:
                return func(*args, **kwargs)

        return wrapper

    return decorator


# @contextlib.contextmanager
# def quietly(hard=False):
#     if hard:
#         original_stdout = sys.stdout
#         captured_output = StringIO()
#         sys.stdout = captured_output
#         yield captured_output
#         sys.stdout = original_stdout
#     else:
#         original_value = current.quietly
#         current.quietly = True
#         current.captured_output = StringIO()
#         try:
#             yield current.captured_output
#         finally:
#             current.quietly = original_value

pdexplorer/_quietly.py
pandichef/pdexplorer
import re
import pandas as pd
from typing import Iterable, Union, List, Any


def _search_lazily(list_to_search: Iterable, search_string: str) -> list:
    """
    Matches columns starting with a string if it's unique in the set of column names.
    For example, 'firstn' would match 'firstname'.  However, if 'firstname' and
    'firstnumber' are two columns, then a NameError is raised.

    :param list_to_search: pandas.DataFrame
    :param search_string: Stata-style list i.e., string with column names separated by a space
    :return: match list

    Example:
    >>> match_list = _search_lazily(list_to_search, 'origfix origfic')
    >>> assert match_list = ['origfixedterm', 'origfico']
    """
    collist = list(
        list_to_search
    )  # valid inputs: pandas.DataFrame, pandas.core.indexes.base.Index, ordinary Python list
    # if type(search_string) == str:
    search_list = search_string.split()

    matchlist = []
    for var in search_list:
        try:
            collist.index(var)  # produces a ValueError if not found
            matchlist.append(var)
        except ValueError:
            matches = [x for x in collist if x.startswith(var)]
            if len(matches) == 1:
                matchlist.append(matches[0])
            elif len(matches) == 0:
                raise NameError("variable " + var + " not found")
            else:
                raise NameError(var + " ambiguous abbreviation")

    return matchlist


def _search_by_range(list_to_search: Iterable, search_string: str) -> list:
    """
    Matches columns based on a range, as specified by the syntax "startvar - endvar"

    :param list_to_search: pandas.DataFrame
    :param search_string: Column range, specified by the syntax "startvar - endvar"
    :return: match list

    Example:
    >>> match_list = _search_by_range(list_to_search, 'origfixedterm-origfico')
    >>> assert match_list = ['origfixedterm', 'origltv', 'origfico']
    """
    collist = list(
        list_to_search
    )  # valid inputs: pandas.DataFrame, pandas.core.indexes.base.Index, ordinary Python list
    # if type(search_string) == str:
    search_list = search_string.split()

    search_string = "".join(search_list)
    search_list = search_string.split("-")

    pos0 = collist.index(search_list[0])
    pos1 = collist.index(search_list[1])
    matchlist = collist[pos0 : pos1 + 1]

    return matchlist


def _search_by_ranges(list_to_search: Iterable, search_string: str) -> list:
    """Handle the case where more than one range is specified"""
    collist = list(
        list_to_search
    )  # valid inputs: pandas.DataFrame, pandas.core.indexes.base.Index, ordinary Python list
    # if type(search_string) == str:
    search_list = search_string.split()

    matchlist = []
    for subsearch_string in search_list:
        thismatchlist = _search_by_range(collist, subsearch_string)
        matchlist.extend(thismatchlist)

    return matchlist


def _search_by_wildcard(list_to_search: Iterable, search_string: str) -> list:
    """
    Matches columns based on * wildcards e.g., "orig*"

    :param list_to_search: pandas.DataFrame
    :param search_string: String containing variables specified by wildcards
    :return: match list
    """
    collist = list(
        list_to_search
    )  # valid inputs: pandas.DataFrame, pandas.core.indexes.base.Index, ordinary Python list

    # if type(search_string) == str:
    search_list = search_string.split()

    relist = []
    for x in search_list:
        x = x.replace("*", ".*")
        relist.append(x)
    restr = "^" + "$|^".join(relist) + "$"
    matchlist = [x for x in collist if re.match(restr, x)]
    return matchlist


def search_iterable(
    list_to_search: Iterable, search_string: str, subtract: bool = False
) -> list:
    """Search any iterable by wildcard, by range, or "lazily"
    """
    collist = list(list_to_search)  # recast iterable as an ordinary list
    # if type(search_string) == str:
    search_list = search_string.split()

    typelist = []
    for x in search_list:
        if x.find("*") > -1:
            typelist.append("w")  # wildcard
        elif x.find("-") > -1:
            typelist.append("r")  # range
        else:
            typelist.append("l")  # lazy

    if len(set(typelist)) == 1 and typelist[0] == "w":
        matchlist = _search_by_wildcard(list_to_search, search_string)
    elif len(set(typelist)) == 1 and typelist[0] == "r":
        matchlist = _search_by_ranges(list_to_search, search_string)
    elif len(set(typelist)) == 1 and typelist[0] == "l":
        matchlist = _search_lazily(list_to_search, search_string)
    else:  # handle mixed search_string (note: this apparently isn't any slower than a 'pure' search_string)
        matchlist = []
        for x in search_list:
            if x.find("*") > -1:
                matchlist.extend(_search_by_wildcard(list_to_search, x))
            elif x.find("-") > -1:
                matchlist.extend(_search_by_ranges(list_to_search, x))
            else:
                matchlist.extend(_search_lazily(list_to_search, x))
    if subtract:
        matchlist = [x for x in collist if x not in matchlist]
    return matchlist

pdexplorer/_search.py
pandichef/pdexplorer
def _stata_slice(in_):
    # convert stata slice (e.g., 1/10) to python slice (e.g., 0:10)
    if isinstance(in_, str) and in_.find("/") > -1:  # Stata's slicing syntax
        in_ = in_.replace("/", ":")
        _in_list = in_.split(":")
        in_ = str(int(_in_list[0]) - 1) + ":" + _in_list[1]
        return in_
    else:
        return in_

pdexplorer/_stata_slice.py
pandichef/pdexplorer
import io
import sys

browser = None


def webbrowser_open(url, join_full_path=True, browser_name="Edge"):
    """
    browser_name: Chrome, Edge, ChromiumEdge, Firefox, Ie, Safari, WebKitGTK, WPEWebKit
    """
    # https://stackoverflow.com/questions/71680422/python-displaying-a-web-site-in-the-same-browser-tab-each-time-webbrowser-open
    global browser
    from selenium import webdriver
    import os

    if join_full_path:
        url = os.path.join(os.getcwd(), url)

    try:
        browser.get(url)
    except:
        # with suppress_stdout():
        try:
            browser = getattr(webdriver, browser_name)()
            browser.get(url)
        except:
            import webbrowser

            webbrowser.open(url)

pdexplorer/_webbrowser.py
pandichef/pdexplorer
from http.server import HTTPServer, SimpleHTTPRequestHandler, ThreadingHTTPServer
import io
from contextlib import redirect_stdout
import threading
import sys
from time import time, sleep
import webbrowser
import numpy as np
import pandas as pd
from ._dataset import current


def make_html(df, num_rows=200, title=""):
    # <meta http-equiv="refresh" content="1">
    result = """
<html>
<head>
<style>

    h2 {
        text-align: center;
        font-family: Helvetica, Arial, sans-serif;
    }
    table { 
        margin-left: auto;
        margin-right: auto;
    }
    table, th, td {
        border: 1px solid black;
        border-collapse: collapse;
    }
    th, td {
        padding: 5px;
        text-align: center;
        font-family: Helvetica, Arial, sans-serif;
        font-size: 90%;
    }
    .wide {
        width: 90%; 
    }
</style>
</head>
<body>
    """
    result += f"<h2>{' '.join(title.split()[:10])}</h2>"
    result += df.head(num_rows).to_html(classes="wide", escape=False, index=False)
    result += """<script>
    let storedTableHTML = '';

    function checkForChanges() {
        fetch('http://localhost:8000/updated_df.html')
            .then(response => response.text())
            .then(newTableHTML => {
                const dataTable = document.getElementById('data-table');
                if (storedTableHTML !== '' && newTableHTML !== storedTableHTML) {
                    // Content has changed, refresh the page
                    location.reload();
                } else {
                    // Update stored table HTML with the new data
                    storedTableHTML = newTableHTML;
                    // Replace the table content with the new HTML
                    dataTable.innerHTML = newTableHTML;
                }
            })
            .catch(error => {
                console.error('Error fetching data:', error);
            });
    }

    // Check for changes every 1 seconds (adjust the interval as needed)
    setInterval(checkForChanges, 1000);
</script></body></html>
"""
    return result


def save_html_file(notify_event, num_rows=200):
    """Function to display the DataFrame when changes occur"""
    previous_df = current.df.copy()
    first_run = True
    while not notify_event.is_set():  # Check the event flag
        current_df = current.df.copy()
        if not current_df.equals(previous_df) or first_run:
            t0 = time()
            with open("updated_df.html", "w") as f:
                f.write(make_html(current_df, num_rows, current.metadata["data_label"]))
            print(f"Saved to file ({time()-t0} ms)")
            previous_df = current_df.copy()
            first_run = False
        sleep(1)


# stop_server = False


def serve_html_file(notify_event):
    # global stop_server
    # stop_server = False

    class SilentRequestHandler(SimpleHTTPRequestHandler):
        def log_message(self, format, *args):
            pass  # Suppress log messages

    httpd = ThreadingHTTPServer(("localhost", 8000), SilentRequestHandler)

    try:
        while True:
            if notify_event.is_set():
                break  # Exit the loop if the event is set
            httpd.handle_request()
    except KeyboardInterrupt:
        pass

    httpd.server_close()
    print("Server stopped gracefully.")


def browse(num_rows=200):
    global notify_event
    global save_thread
    global serve_thread
    if not current.browse_turned_on:
        # # Create an Event to notify the display thread of changes #
        notify_event = threading.Event()
        # Create a save_thread to save the DataFrame when changes occur #
        save_thread = threading.Thread(
            target=save_html_file, args=(notify_event, num_rows,)
        )
        save_thread.daemon = True  # prevent console from blocking
        current.browse_turned_on = True
        save_thread.start()
        # Create a serve_thread to create a light weight web server #
        # serve_notify_event = threading.Event()
        serve_thread = threading.Thread(target=serve_html_file, args=(notify_event,))
        serve_thread.daemon = True  # prevent console from blocking
        serve_thread.start()
    else:
        print("Already turned on.")
    from ._webbrowser import webbrowser_open

    webbrowser_open("http://localhost:8000/updated_df.html", join_full_path=False)


def browse_off():
    # global stop_server
    notify_event.set()
    save_thread.join()
    serve_thread.join()
    current.browse_turned_on = False
    # stop_server = True
    import os

    sleep(1)
    os.remove("updated_df.html")


# atexit.register doesn't work to clean up the thread... but this does: #
# https://stackoverflow.com/questions/58910372/script-stuck-on-exit-when-using-atexit-to-terminate-threads #
# def monitor_thread():
#     main_thread = threading.main_thread()
#     main_thread.join()
#     browse_notify_event.set()
#     browse_thread.join()


# monitor = threading.Thread(target=monitor_thread)
# monitor.daemon = True
# monitor.start()

pdexplorer/browse.py
pandichef/pdexplorer
def cf(varlist, using, package="ydata_profiler"):
    from ._profile import ydata_profile, sweetviz_profile

    if package == "ydata_profiler":
        ydata_profile(varlist=varlist, compare_to=using)
    elif package == "sweetviz":
        sweetviz_profile(varlist=varlist, compare_to=using)
    else:
        raise Exception(f"Package {package} not found.")

pdexplorer/cf.py
pandichef/pdexplorer
from ._dataset import current, Dataset


def clear():
    current.clear()


def clearall():
    current.clearall()

pdexplorer/clear.py
pandichef/pdexplorer
import numpy as np
from ._dataset import current
from ._print import _print
from ._commandarg import parse, parse_options


def collapse(commandarg: str) -> None:
    _ = parse(commandarg)

    weight = _["weight"]
    weight = weight.split("=")[1] if weight else None

    by = _["by"] if _["by"] else None

    # if _["options"]:
    #     parsed_options = parse_options(_["options"])
    #     if parsed_options["by"]:
    #         by = parsed_options["by"]
    #     else:
    #         by = None
    # else:
    #     by = None

    clist = _["anything"]

    clist_as_list = clist.split()
    # print(clist_as_list)
    # assert len(clist_as_list) == len(
    #     set(clist_as_list)
    # ), "Duplicate names found.  Create appropriately named columns first before collapsing."

    # df = current.df
    if not weight:
        current._df["weight"] = 1
        weight = "weight"
    if not by:
        current._df["by"] = "_all"
        by = "by"

    fnc = {
        # "mean": lambda x: (x * df[weight]).sum() / (df[weight]).sum(), # Denominator didn't work here
        "mean": lambda x: (x * current._df.loc[x.index, weight]).sum()  #  From ChatGPT
        / (current._df.loc[x.index, weight]).sum(),  # type:ignore
        "median": lambda x: x.median(),
        "sum": lambda x: (x * current._df[weight]).sum(),
        "rawsum": lambda x: x.sum(),
        "count": lambda x: x.count(),
        "max": lambda x: x.max(),
        "min": lambda x: x.min(),
    }

    # parse clist
    agg_functions = {}
    stat = ""
    for item in clist_as_list:
        if item[0] == "(" and item[-1] == ")":
            stat = item[1:-1]
        else:
            agg_functions[item] = fnc[stat]
        # print(stat)

    if not weight:
        del current._df["weight"]

    current.df = current._df.groupby(by).agg(agg_functions).reset_index()

    _print(current._df)

pdexplorer/collapse.py
pandichef/pdexplorer
import json
import requests
from typing import Optional
import datetime
import pandas as pd

# from datetime import datetime

# Documented here: #
# https://developers.binance.com/docs/binance-trading-api/futures#get-funding-rate-history #
# root_url = "https://fapi.binance.com/fapi/v1/fundingRate"
root_url = "https://fapi.binance.com/"


def fapi_v1_fundingRate_wrapper(
    symbol: Optional[str] = None,
    start_time: Optional[int] = None,
    end_time: Optional[int] = None,
    limit: Optional[int] = None,
) -> list:
    url_path = "fapi/v1/fundingRate"
    if limit:
        assert limit <= 1000, f"Per {url_path}, limit must be <=1000"
    query = []
    query.append("symbol=" + symbol if symbol else "")
    query.append("startTime=" + str(start_time) if start_time else "")
    query.append("endTime=" + str(end_time) if end_time else "")
    query.append("limit=" + str(limit) if limit else "")
    query = list(filter(None, query))
    query = "&".join(query)
    url = root_url + url_path + "?" + query
    res = requests.get(url)
    return json.loads(res.text)


def to_dataframe(result: list) -> pd.DataFrame:
    df = pd.DataFrame(result)
    df["fundingTime"] = pd.to_datetime(df["fundingTime"], unit="ms")
    df["fundingRate"] = pd.to_numeric(df["fundingRate"], errors="coerce")
    return df


now = datetime.datetime.now()
end_time = int(now.timestamp() * 1000)
start_time = int((now - datetime.timedelta(days=3)).timestamp() * 1000)

result = fapi_v1_fundingRate_wrapper("ETHUSDT", start_time, end_time, 1000)
df = to_dataframe(result)

"""
# Original 
# Source:
# https://steemit.com/python/@marketstack/how-to-download-historical-price-data-from-binance-with-python
# https://github.com/binance/binance-spot-api-docs/blob/master/rest-api.md#klinecandlestick-data
import datetime as dt  # for dealing with times
import json  # for parsing what binance sends back to us
import hashlib
from json import JSONDecodeError
from types import SimpleNamespace  # for building urls
from typing import List, Union, Optional
import requests  # for making http requests to binance
from requests import Session, Request, Response
import pandas as pd
import numpy as np
from pandas import DataFrame

# from ...utils import pure

# root_url = "https://api.binance.com/api/v3/klines"
root_url = "https://fapi.binance.com/fapi/v1/fundingRate"
# from .urls import binance_funding_rate as root_url

# @pure
def _build_query(symbol: str | None = None, limit: int = 1000) -> Request:
    limit = int(limit)
    if symbol is None:
        url = root_url + "?limit=" + str(limit)
    else:
        url = root_url + "?symbol=" + symbol + "&limit=" + str(limit)
    return Request("GET", url)


def _make_request(request: Request) -> Response:
    s = Session()
    prepped_ = s.prepare_request(request)
    resp = s.send(prepped_)
    # print("#######################################################")
    # print("funding_rate.py: " + resp.text + "...")
    # print("#######################################################")
    return resp


# @pure
def _check_response(
    response: Response | SimpleNamespace, show_reponse: bool = False
) -> None:
    # nested function
    def _wrap(text: str, show_reponse: bool = show_reponse) -> str:
        if show_reponse:
            return __name__ + text + "\n" + response.text
        else:
            return __name__ + text

    try:
        obj = json.loads(response.text)
    except JSONDecodeError:
        raise AssertionError(_wrap(" could not be parsed (JSONDecodeError)"))
    assert type(obj) == list, _wrap(": response is not a list")
    assert type(obj[0]) == dict, _wrap(": response is not a list of objects")
    assert len(obj[0]) == 3, _wrap(": response is not a list of 3 numbers")
    return None


# @pure
def _make_dataframe(response: Response | SimpleNamespace) -> DataFrame:
    df = pd.DataFrame(json.loads(response.text))
    # df["fundingTime"] = pd.to_datetime(df["fundingTime"], unit="s")
    df["fundingTime"] = [dt.datetime.fromtimestamp(x / 1000) for x in df["fundingTime"]]
    df["fundingRate"] = df["fundingRate"].astype("float64")
    return df


def get_funding_rate(symbol: str | None = None, limit: int = 1000) -> DataFrame:
    request = _build_query(symbol=symbol, limit=limit)
    response = _make_request(request)
    _check_response(response, True)
    df = _make_dataframe(response)
    return df


def get_enhanced_funding_rate(
    symbol: str | None = None, limit: int = 1000
) -> DataFrame:
    # Do nothing for now
    df = get_funding_rate(symbol, limit)
    max_timestamp = df.fundingTime.max()
    df["is_latest"] = np.where(df["fundingTime"] == max_timestamp, True, False)
    return df


def extract_current_funding_rate(symbol: str | None = None) -> float:
    df = get_enhanced_funding_rate(symbol, 1)
    # print(df)
    return df["fundingRate"][0]


################################################################
# Applications


################################################################
if __name__ == "__main__":
    # df = get_enhanced_funding_rate()
    # df = get_klines("ETHUSDT", 20, "1d")
    # from ...utils import browse

    # browse(df=df)
    # vol = get_volatility("GRTUSDT", 30)
    # print(vol)
    # print(df)
    # res = extract_current_funding_rate("ETHUSDT")
    res = get_enhanced_funding_rate("ETHUSDT")
    print(res)
"""


pdexplorer/data/binance/fapi_v1_fundingRate.py
pandichef/pdexplorer
import os
import json
from ..utils import tothegraph


def get_pool_page_queries():
    # print(os.path.dirname(__file__))
    with open(os.path.join(os.path.dirname(__file__), "pool_page.json"), "r") as f:
        pool_page = json.loads(f.read())
    return pool_page


# 0: TS daily
# 1: cf 6 other pools
# 2: cf 50 other pools
# 3: transactions
# 4: factory summary
# 5: block number (scalar value) i.e., 18245306 use in [12, 15, 20, 24, 28, 32, 34, 38]
# 6: cf 7 other pools
# 7: cf 50 other tokens
# 8: transactions
# 9: error
# 10: empty
# 11: block number (scalar value) i.e., 18238166 use in [13, 16, 21, 25, 29, 31, 35, 39]
# 12: cf 6 other pools
# 13: cf 6 other pools
# 14: cf 6 other pools
# 15: empty
# 16: empty

pdexplorer/data/thegraph/uniswap_v3/pool_page.py
pandichef/pdexplorer
import json

# from json import JSONDecodeError
# from typing import Optional
import pandas as pd


def extract_api_requests(
    page="https://info.uniswap.org/#/pools/0x88e6a0c2ddd26feeb64f039a2c41296fcb3f5640",
    api_prefix="https://api.thegraph.com/subgraphs/name/",
    wait_for: int = 20,
    filter_address="0x88e6a0c2ddd26feeb64f039a2c41296fcb3f5640",
    output_file="pool_page.json",
):
    """
    This automates the process of extracting API requests from a page.
    The manual process is as follows: dev tools -> Network tab -> Fetch/XHR
    """
    from seleniumwire import webdriver  # Import from seleniumwire
    from selenium.webdriver.edge.options import Options

    # Create a new instance of the Chrome driver
    edge_options = Options()
    edge_options.use_chromium = True
    edge_options.add_argument("--headless")  # Optional: Run Edge in headless mode
    # driver = webdriver.Edge()
    try:
        driver = webdriver.Chrome(options=edge_options)
    except:
        driver = webdriver.Edge(options=edge_options)

    # Go to the Google home page
    driver.get(page)

    import time

    time.sleep(wait_for)

    list_of_requests0 = driver.requests.copy()
    list_of_requests1 = []

    for request in list_of_requests0:
        if request.url.startswith(api_prefix):
            list_of_requests1.append(request)

    print(
        f"Found {len(list_of_requests0)} requests; only {len(list_of_requests1)} are valid API requests."
    )
    # print(len(list_of_requests0))
    # print(len(list_of_requests1))

    list_of_dicts = []
    for i, obj in enumerate(list_of_requests1):
        # print(i)
        # print(repr(obj))
        new_dict = {}
        # try:
        body = json.loads(obj.body)
        new_dict["query"] = body["query"]
        if isinstance(body["variables"], str):
            new_dict["variables"] = json.loads(body["variables"])
        else:
            new_dict["variables"] = body["variables"]
        new_dict["endpoint"] = obj.url
        # new_dict["fnc"] = lambda x: x  # pd.DataFrame.from_records(x)
        list_of_dicts.append(new_dict)
        # except JSONDecodeError as e:
        #     print(obj)
        #     raise Exception("stop running")

    assert len(list_of_dicts) == len(list_of_requests1)

    # remove if its a different address #
    if filter_address:
        list_of_dicts2 = []
        for i, this_dict in enumerate(list_of_dicts):
            if this_dict["variables"].get("address") is None:
                list_of_dicts2.append(this_dict)
            elif (
                this_dict["variables"].get("address") is not None
                and this_dict["variables"]["address"] == filter_address
            ):
                list_of_dicts2.append(this_dict)
            else:
                pass
        print(
            f"Found {len(list_of_dicts)} valid API requests; only {len(list_of_dicts2)} are related to address {filter_address}."
        )
    else:
        list_of_dicts2 = list_of_dicts

    if output_file:
        with open(output_file, "w") as f:
            f.write(json.dumps(list_of_dicts2))

    return list_of_dicts2
    # return list_of_dicts


def _tothegraph(
    query: str,
    variables,
    endpoint: str = "https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v3",
) -> dict:
    import json
    from gql import Client, gql
    from gql.transport.aiohttp import AIOHTTPTransport
    import pandas as pd

    # Select your transport with a defined url endpoint
    transport = AIOHTTPTransport(url=endpoint)

    # Create a GraphQL client using the defined transport
    client = Client(transport=transport, fetch_schema_from_transport=True)

    # Provide a GraphQL query
    # gql apparently validates, but not thegraph server... #
    query = query.replace("Bytes!", "String!")

    # Execute the query on the transport
    if variables:
        # try:
        # result = client.execute(gql(query), variable_values=json.loads(variables))
        # except TypeError:
        result = client.execute(gql(query), variable_values=variables)
    else:
        result = client.execute(gql(query))
    # print(result)
    return result


def tothegraph(query_dict: dict, key_number=0) -> pd.DataFrame:
    """Send query to API server and return pd.DataFrame"""
    query = query_dict["query"]
    variables = query_dict["variables"]
    endpoint = query_dict["endpoint"]
    # fnc = query_dict["fnc"]
    response_dict = _tothegraph(query, variables, endpoint)
    if key_number is not None:
        key_name = list(response_dict.keys())[key_number]
        df = pd.DataFrame.from_records(response_dict[key_name])
    else:
        # key_name = list(response_dict.keys())[key_number]
        df = pd.DataFrame.from_records(response_dict)
    # if len(df) == 0:
    #     df = pd.DataFrame.from_records(response_dict)

    # df.to_clipboard()
    try:
        df["date"] = pd.to_datetime(df["date"], unit="s")
    except KeyError:
        pass

    for col in df.columns:
        if col != "date":
            try:
                df[col] = pd.to_numeric(df[col])
            except (ValueError, TypeError):
                pass

    return df


def searchframes(list_of_dicts, column_name="volumeUSD"):
    """Send query to API server and search for column values"""
    from gql.transport.exceptions import TransportQueryError

    for i, this_dict in enumerate(list_of_dicts):
        try:
            df = tothegraph(this_dict)
            if "date" in df.columns:
                df = df.sort_values("date", ascending=False)
            if column_name in df.columns:
                print(
                    i,
                    ": ",
                    df[column_name][0],
                    f""" ({df['date'][0] if "date" in df.columns else '<NA>'})""",
                )
        except TransportQueryError as e:
            pass


def searchqueries(list_of_dicts, query_type="poolDayDatas"):
    """Search for column names in API request string"""
    query_type = query_type.lower()
    match_list = []
    for i, this_dict in enumerate(list_of_dicts):
        if query_type in this_dict["query"].lower():
            print(
                "[",
                i,
                "]\nQUERY:\n",
                this_dict["query"],
                "\nVARIABLES:\n",
                this_dict["variables"],
                "\n",
            )
            match_list.append(i)
    return match_list

pdexplorer/data/thegraph/utils.py
pandichef/pdexplorer
from ._search import search_iterable
from ._dataset import current
from ._print import _print


def describe(varlist=None):
    df = current.df
    if varlist:
        labels = search_iterable(df.columns, varlist)
        inverted_labels = [x for x in df.columns if x not in labels]
        df = df.drop(labels=inverted_labels, axis=1, inplace=False)
    _print(df.info())

pdexplorer/describe.py
pandichef/pdexplorer
from ._dataset import current


def doedit(filename: str) -> None:
    import subprocess

    subprocess.run(f"code {filename}", shell=True)
    current.active_python_script = filename


# def _run_active_python_script():
#     os.subprocess(f'ipython -c "%run {current.active_python_script}"', s)
###################################################################################
def check_script_for_input(script_path):
    """
    input() crashes when used with a hotkey
    This is true for both the keyboard and pynput packages
    """
    import ast

    class InputNotAllowedException(Exception):
        pass

    with open(script_path, "r") as script_file:
        script_source = script_file.read()

    # Parse the script source code into an Abstract Syntax Tree (AST)
    try:
        tree = ast.parse(script_source)
    except SyntaxError as e:
        print(f"Syntax error in the script: {e}")
        return

    # Walk through the AST to check for 'input' calls
    for node in ast.walk(tree):
        if (
            isinstance(node, ast.Call)
            and isinstance(node.func, ast.Name)
            and node.func.id == "input"
        ):
            # raise InputNotAllowedException(
            #     "The 'input()' function is not allowed in this script"
            # )
            return False
    return True


def run_active_python_script() -> None:
    from IPython.core.getipython import get_ipython

    if current.active_python_script:
        if check_script_for_input(current.active_python_script):
            print(f"Running {current.active_python_script}.")
            get_ipython().run_line_magic("run", current.active_python_script)
        else:
            print(
                f"Found input() in {current.active_python_script}, which causes ipython console to crash.\nRewrite the script or run the file explicitly."
            )
    else:
        print("No active python script. Run doedit command first.")

pdexplorer/doedit.py
pandichef/pdexplorer
from ._search import search_iterable
from ._dataset import current
from ._commandarg import parse
from ._print import _print
from ._dataset import current
from ._search import search_iterable
from ._print import _print
from ._stata_slice import _stata_slice
from ._dataset import current
from ._print import _print


def dropin(range: str) -> None:
    range = _stata_slice(range)
    current.df = eval(f"current.df.drop(current.df.index[{range}])")
    _print(current.df)


def dropif(condition: str) -> None:
    current.df.query("~(" + condition + ")", inplace=True)
    _print(current.df)


def drop(commandarg: str) -> None:
    _ = parse(commandarg)

    if _["anything"]:
        assert (
            _["in"] is None and _["if"] is None
        ), "drop takes either a varlist or in/if, but not both"
        varlist = search_iterable(current._df.columns, _["anything"])
        current._df.drop(labels=varlist, axis=1, inplace=True)
    elif _["in"] or _["if"]:
        assert (
            _["anything"] is None
        ), "drop takes either a varlist or in/if, but not both"
        if _["in"]:
            dropin(_["in"])
        if _["if"]:
            print("YES")
            dropif(_["if"])
    else:
        raise Exception("drop: Missing Arguments")

    current._df.reset_index(inplace=True, drop=True)
    _print(current._df)

pdexplorer/drop.py
pandichef/pdexplorer
# under construction: additional egen functions must be added over time
import numpy as np
from ._dataset import current
from ._print import _print


def egen(assignment, by=None):
    """
    Generates a new function variable given input of the form:
    egen("newvar = fnc(arguments)")

    see https://www.stata.com/manuals/degen.pdf
    """
    assignment_list = assignment.split()
    assert assignment_list[1] == "=", 'egen assignment must contain "=".'
    newvar = assignment_list[0]
    fnc_and_args = "".join(assignment_list[2:])
    fnc_and_args_list = fnc_and_args.split("(")
    egen_fnc = fnc_and_args_list[0]
    assert len(fnc_and_args_list) == 2, "Invalid expression."
    arguments = fnc_and_args_list[1][:-1]

    if not by:
        current._df["by"] = "_all"
        by = "by"

    if egen_fnc == "rank":
        current._df[newvar] = current._df.groupby(by)[arguments].rank(method="average")
    else:
        # mean, count, median, min, max
        current._df[newvar] = current._df.groupby(by)[arguments].transform(egen_fnc)

    if by == "by":
        del current._df["by"]

    _print(current._df)

pdexplorer/egen.py
pandichef/pdexplorer
import os
import requests


def bard(prompt: str = "How is the weather today in seoul?"):
    # See https://github.com/dsdanielpark/Bard-API/issues/155
    # This is a really a big hack and barely works
    from bardapi import Bard, SESSION_HEADERS

    __Secure1PSID = os.environ["__Secure-1PSID"]
    __Secure1PSIDCC = os.environ["__Secure-1PSIDCC"]
    __Secure1PSIDTS = os.environ["__Secure-1PSIDTS"]

    session = requests.Session()
    token = __Secure1PSID
    # print(token)
    session.cookies.set("__Secure-1PSID", token)
    session.cookies.set("__Secure-1PSIDCC", __Secure1PSIDCC)
    session.cookies.set("__Secure-1PSIDTS", __Secure1PSIDTS)
    session.headers = SESSION_HEADERS

    bard = Bard(token=token, session=session)
    res = bard.get_answer(prompt)
    return res["content"]

pdexplorer/experimental/bard.py
pandichef/pdexplorer
import io
import sys
from contextlib import contextmanager
from .._dataset import current
from .._print import _print


@contextmanager
def capture_stdout():
    # Create a StringIO object to capture stdout
    old_stdout = sys.stdout
    sys.stdout = io.StringIO()

    try:
        yield sys.stdout
    finally:
        # Restore the original stdout
        sys.stdout = old_stdout


def chatgpt(
    prompt: str = "Can you summarize this data?  Who is the data provider for this data?",
    head=100,
    use_variable_labels=False,
    openai_model="gpt-3.5-turbo",
):
    # See https://platform.openai.com/account/billing/overview
    import os
    import openai

    if use_variable_labels:
        _current_df = current.df.rename(columns=current.metadata["variable_labels"])
    else:
        _current_df = current.df

    with capture_stdout() as captured_output:
        _current_df.info()

    # print(captured_output.getvalue())
    # print(captured_output.getvalue())

    if head:
        prompt = (
            "Data:\n"
            + _current_df.head(head).to_csv()
            + "\n\n"
            + captured_output.getvalue()
            + f"\n\n{prompt}"
        )
    else:
        prompt = (
            "Data:\n"
            + _current_df.to_csv()
            + "\n\n"
            + captured_output.getvalue()
            + f"\n\n{prompt}"
        )

    openai.api_key = os.getenv("OPENAI_API_KEY")

    completion = openai.ChatCompletion.create(
        model=openai_model, messages=[{"role": "user", "content": prompt,}],
    )

    _print(completion.choices[0].message.content)
    return completion.choices[0].message.content

pdexplorer/experimental/chatgpt.py
pandichef/pdexplorer
from ..webuse import webuse
from ..keep import keep
from .._dataset import current
from .._print import _print
from .._quietly import quietly

# example_indices_full = [40, 80, 120]
# example_index_to_summarize = 200


def flan_t5_base():
    # see https://www.coursera.org/learn/generative-ai-with-llms/lecture/wno7h/lab-1-walkthrough
    with quietly():
        from transformers import AutoModelForSeq2SeqLM
        from transformers import AutoTokenizer
        from transformers import GenerationConfig

        model_name = "google/flan-t5-base"

        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
        tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)

        webuse("knkarthick/dialogsum", "datasets", use_local=True)
        keep('if split =="test"')
        # current.df.reset_index(inplace=True)
        keep("if index==40 or index==200")
        # observation1 = current.df["dialogue"][0]

        # def tokenizer_lambda(sentence):
        #     return tokenizer(sentence, return_tensors="pt")

        current.df["encoded"] = current.df["dialogue"].apply(
            lambda x: tokenizer(x, return_tensors="pt")["input_ids"]  # type: ignore
        )

        current.df["decoded"] = current.df["encoded"].apply(
            lambda x: tokenizer.decode(x[0], skip_special_tokens=True)
        )

        # print(current.df["dialogue"][0].replace("\n", " "))
        # print(current.df["decoded"][0])
        assert current.df["dialogue"][0].replace("\n", " ") == current.df["decoded"][0]

        #  current.df["decoded"] = current.df["encoded"].apply(

        current.df["yhat"] = current.df["encoded"].apply(
            lambda x: tokenizer.decode(
                model.generate(x, max_new_tokens=50)[0], skip_special_tokens=True
            )
        )

        current.df["prompt"] = current.df.apply(
            lambda row: "Dialogue:\n{}\n\nSummary:\n{}".format(
                row["dialogue"], row["summary"]
            ),
            axis=1,
        )

        print(current.df["prompt"][0])

    # print(observation1)
    # print()
    # print(observation2)

    # print(current.df)

pdexplorer/experimental/flan_t5_base.py
pandichef/pdexplorer
import pandas as pd
from .._dataset import current
from .._print import _print


def generate_split(*args, **kwargs):
    from sklearn.model_selection import train_test_split

    train, test = train_test_split(current.df, *args, **kwargs)
    train["split"] = "train"  # type: ignore
    test["split"] = "test"  # type: ignore
    current.df = pd.concat([train, test], axis=0).reset_index(drop=True)  # type: ignore
    _print(current.df)

pdexplorer/experimental/generate_split.py
pandichef/pdexplorer
from .._search import search_iterable
from .._dataset import current


def nnlinear(varlist: str):
    # https://pytorch.org/tutorials/beginner/pytorch_with_examples.html#pytorch-nn
    import torch
    import math

    # Create Tensors to hold input and outputs.
    x = torch.linspace(-math.pi, math.pi, 2000)
    y = torch.sin(x)

    # For this example, the output y is a linear function of (x, x^2, x^3), so
    # we can consider it as a linear layer neural network. Let's prepare the
    # tensor (x, x^2, x^3).
    p = torch.tensor([1, 2, 3])
    xx = x.unsqueeze(-1).pow(p)

    varlist_as_list = varlist.split()
    yvar = varlist_as_list[0]
    xvars = varlist_as_list[1:]
    xvars = search_iterable(current.df.columns, " ".join(xvars))

    X_numpy = current.df.dropna()[xvars].values
    y_numpy = current.df.dropna()[[yvar]].values

    xx = torch.tensor(X_numpy)
    y = torch.tensor(y_numpy)

    print(y.shape)
    print(xx.shape)
    print(y)
    print(xx)

    # In the above code, x.unsqueeze(-1) has shape (2000, 1), and p has shape
    # (3,), for this case, broadcasting semantics will apply to obtain a tensor
    # of shape (2000, 3)

    # Use the nn package to define our model as a sequence of layers. nn.Sequential
    # is a Module which contains other Modules, and applies them in sequence to
    # produce its output. The Linear Module computes output from input using a
    # linear function, and holds internal Tensors for its weight and bias.
    # The Flatten layer flatens the output of the linear layer to a 1D tensor,
    # to match the shape of `y`.
    model = torch.nn.Sequential(torch.nn.Linear(3, 1), torch.nn.Flatten(0, 1))

    # The nn package also contains definitions of popular loss functions; in this
    # case we will use Mean Squared Error (MSE) as our loss function.
    loss_fn = torch.nn.MSELoss(reduction="sum")

    learning_rate = 1e-6
    for t in range(2000):

        # Forward pass: compute predicted y by passing x to the model. Module objects
        # override the __call__ operator so you can call them like functions. When
        # doing so you pass a Tensor of input data to the Module and it produces
        # a Tensor of output data.
        y_pred = model(xx)

        # Compute and print loss. We pass Tensors containing the predicted and true
        # values of y, and the loss function returns a Tensor containing the
        # loss.
        loss = loss_fn(y_pred, y)
        if t % 100 == 99:
            print(t, loss.item())

        # Zero the gradients before running the backward pass.
        model.zero_grad()

        # Backward pass: compute gradient of the loss with respect to all the learnable
        # parameters of the model. Internally, the parameters of each Module are stored
        # in Tensors with requires_grad=True, so this call will compute gradients for
        # all learnable parameters in the model.
        loss.backward()

        # Update the weights using gradient descent. Each parameter is a Tensor, so
        # we can access its gradients like we did before.
        with torch.no_grad():
            for param in model.parameters():
                param -= learning_rate * param.grad

    # You can access the first layer of `model` like accessing the first item of a list
    linear_layer = model[0]

    # For linear layer, its parameters are stored as `weight` and `bias`.
    print(
        f"Result: y = {linear_layer.bias.item()} + {linear_layer.weight[:, 0].item()} x + {linear_layer.weight[:, 1].item()} x^2 + {linear_layer.weight[:, 2].item()} x^3"
    )

    # import torch
    # import torch.nn as nn
    # import numpy as np
    # from sklearn import datasets
    # import matplotlib.pyplot as plt

    # varlist_as_list = varlist.split()
    # yvar = varlist_as_list[0]
    # xvars = varlist_as_list[1:]
    # xvars = search_iterable(current.df.columns, " ".join(xvars))

    # X_numpy = current.df.dropna()[xvars].values
    # y_numpy = current.df.dropna()[yvar].astype(float).values

    # # 0) Prepare data
    # # X_numpy, y_numpy = datasets.make_regression(
    # #     n_samples=100, n_features=2, noise=20, random_state=4
    # # )
    # print(X_numpy)
    # print(y_numpy)

    # # cast to float Tensor
    # X = torch.from_numpy(X_numpy.astype(np.float32))
    # y = torch.from_numpy(y_numpy.astype(np.float32))
    # y = y.view(y.shape[0], 1)

    # n_samples, n_features = X.shape

    # # 1) Model
    # # Linear model f = wx + b
    # input_size = n_features
    # output_size = 1
    # model = nn.Linear(input_size, output_size)

    # # 2) Loss and optimizer
    # learning_rate = 0.01

    # criterion = nn.MSELoss()
    # optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

    # # 3) Training loop
    # num_epochs = 100
    # for epoch in range(num_epochs):
    #     # Forward pass and loss
    #     y_predicted = model(X)
    #     loss = criterion(y_predicted, y)

    #     # Backward pass and update
    #     loss.backward()
    #     optimizer.step()

    #     # zero grad before new step
    #     optimizer.zero_grad()

    #     if (epoch + 1) % 10 == 0:
    #         print(f"epoch: {epoch+1}, loss = {loss.item():.4f}")

    # # Plot
    # # predicted = model(X).detach().numpy()

    # # plt.plot(X_numpy, y_numpy, "ro")
    # # plt.plot(X_numpy, predicted, "b")
    # # plt.show()

pdexplorer/experimental/nnlinear.py
pandichef/pdexplorer
# TODO: support multiple variables e.g.,  scatter yvar1 yvar2 xvar
# Deprecated: use altair_mapper instead
from .._dataset import current
from .._search import search_iterable
import warnings
from .._commandarg import parse


def scatter(commandarg: str, byvar=None) -> None:
    import matplotlib.pyplot as plt

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        import seaborn as sns  # produces a bunch of warnings during tests

    sns.set()

    _ = parse(commandarg)
    df = current.df
    # try:
    varlist = search_iterable(df.columns, _["anything"])
    # fig, ax = plt.subplots()
    assert len(varlist) == 2, "Scatterplot only supports two variables"
    # ax.scatter(labels[1], labels[0], data=df)
    scatterplot = sns.scatterplot(x=varlist[1], y=varlist[0], data=df)

    try:
        scatterplot.set(
            xlabel=current.metadata["variable_labels"][varlist[1]],
            ylabel=current.metadata["variable_labels"][varlist[0]],
        )
    except KeyError:
        scatterplot.set(xlabel=varlist[1], ylabel=varlist[0])

    if byvar:
        for index, row in df.iterrows():
            plt.text(row[varlist[1]], row[varlist[0]], row[byvar], fontsize=12)
    plt.show()

pdexplorer/experimental/scatter_seaborn.py
pandichef/pdexplorer
import re
import subprocess
from .._dataset import current
from .chatgpt import chatgpt

streamlit_script_filename = "tmpqbxcec2k.py"
dataframe_csv_filename = "tmpqbxcec2k.csv"


def streamlit(
    prompt: str = """Present this data in the best way that will impress my CEO.""",
    chart_library=None,
    use_variable_labels=False,
    openai_model="gpt-4",
):
    if use_variable_labels:
        _current_df = current.df.rename(columns=current.metadata["variable_labels"])
    else:
        _current_df = current.df

    _current_df.to_csv(dataframe_csv_filename, index=False)
    # _current_df.to_csv(dataframe_csv_filename, index=True)
    full_prompt = f"""Can you write a streamlit script.  Double check that the streamlit syntax is valid.  {prompt} Assume the data is stored in a csv file called {dataframe_csv_filename}."""

    if chart_library:
        full_prompt = (
            f"""  Wherever possible, use the {chart_library} for all charts."""
        )

    if current.metadata["data_label"]:
        full_prompt += f" Use {current.metadata['data_label']} as the script title."

    chatgpt_response_text = chatgpt(
        full_prompt, use_variable_labels=use_variable_labels, openai_model=openai_model
    )

    start_position = chatgpt_response_text.find("```python\n")
    end_position = chatgpt_response_text.find("```\n")

    python_script_text = "\n".join(
        chatgpt_response_text[start_position:end_position].split("\n")[1:]
    )

    with open(streamlit_script_filename, "w") as file:
        file.write(python_script_text)

    print(python_script_text)

    subprocess.run(
        f"streamlit run {streamlit_script_filename} --deprecation.showPyplotGlobalUse=false",
        shell=True,
    )

    # return python_script_text

pdexplorer/experimental/streamlit.py
pandichef/pdexplorer
# Inspiration: https://xlsxwriter.readthedocs.io/working_with_pandas.html
# Note the complexity in handling the header in pandas
import pandas as pd
import subprocess
import os
from ._dataset import current
from typing import Optional


# def browse_in_excel(
def export(
    path: Optional[str] = None,
    limit: Optional[int] = None,
    run: bool = True,
    percentage_columns: Optional[list] = None,
    autofit_columns: Optional[list] = None,
) -> None:
    df = current.df

    if not path:
        # set default file i.e., tmp/easy_xlsxwriter.xlsx
        import tempfile

        path = os.path.join(tempfile.gettempdir(), "easy_xlsxwriter.xlsx")

    writer = pd.ExcelWriter(path=path, engine="xlsxwriter")  # type: ignore
    if limit:
        df.head(limit).to_excel(writer, sheet_name="Sheet1", startrow=1, header=False)
    else:
        df.to_excel(writer, sheet_name="Sheet1", startrow=1, header=False)
    workbook = writer.book
    worksheet = writer.sheets["Sheet1"]

    default_format = workbook.add_format(  # type: ignore
        {
            "num_format": "#,##0.00",
            "align": "center",
            # 'shrink': True,
        }
    )
    for column in df.columns:
        this_column_position = list(df.columns).index(column) + 1
        worksheet.set_column(
            this_column_position, this_column_position, cell_format=default_format
        )

    percentage_format = workbook.add_format({"num_format": "0%", "align": "center"})  # type: ignore
    if percentage_columns:
        for column in percentage_columns:
            this_column_position = list(df.columns).index(column) + 1
            worksheet.set_column(
                this_column_position,
                this_column_position,
                cell_format=percentage_format,
            )

    # format header
    header_format = workbook.add_format(  # type: ignore
        {
            "bold": True,
            "text_wrap": True,
            "align": "center",
            "valign": "top",
            "fg_color": "#D7E4BC",
            "border": 1,
        }
    )
    for col_num, value in enumerate(df.columns.values):
        worksheet.write(0, col_num + 1, value, header_format)
    worksheet.write(0, 0, "Ticker", header_format)
    worksheet.freeze_panes(1, 1)

    # autofit
    if autofit_columns:
        for column in autofit_columns:
            column_width = df[column].str.len().max()
            this_column_position = list(df.columns).index(column) + 1
            worksheet.set_column(
                this_column_position, this_column_position, width=column_width
            )

    # save excel file
    worksheet.set_zoom(75)
    writer.close()  # type: ignore
    if run:
        subprocess.run(f"start excel {path}", shell=True)

pdexplorer/export.py
pandichef/pdexplorer
# UNDER CONSTRUCTION
# BC35 calculator: yield -> price
# https://github.com/pandichef/magicpandas/blob/master/magicpandas/frame.py#L479
# https://github.com/pandichef/pandicake/blob/master/pandicake/pdmonkey/pricegen.py
import pandas as pd
from copy import copy, deepcopy

# import numpy as np
from .._dataset import current
from .._print import _print


def _bc35_price(
    beyield,
    coupon,
    cpr,
    age,
    origfixedterm,
    origterm,
    dayofmonth=1,
    delay=47,
    servicing_fee=0,
    root=None,
):  # 47 is market standard according to Goldmach Sachs
    """This implements BC35 in core Python (vs columnwise in a pandas.DataFrame)
    """
    epsilon = servicing_fee / 1200

    r = coupon / 1200
    y = (1 + beyield / 200) ** (1 / 6) - 1
    s = 1 - (1 - cpr / 100) ** (1 / 12)

    m = origfixedterm - age
    n = origterm - age

    qn = 1 - 1 / (1 + r) ** n
    l1 = (r * (1 - s) - epsilon + s * (1 + r)) / qn  # lambda 1
    l2 = (s * (1 + r) - epsilon) * (1 - 1 / qn)  # lambda 2
    l3 = (1 - 1 / qn) * ((1 + r) * (1 - s)) ** m + (1 / qn) * (1 - s) ** m  # lambda 3

    pvl1 = (l1 / (y + s)) * (1 - ((1 - s) / (1 + y)) ** m)  # PV of lamda 1 term
    pvl2 = (l2 / (y - (1 + r) * (1 - s) + 1)) * (
        1 - ((1 + r) * (1 - s) / (1 + y)) ** m
    )  # PV of lamda 2 term
    pvb = l3 / (1 + y) ** m  # PV of the balloon term
    price = 100 * (pvl1 + pvl2 + pvb)

    # Mod Durn
    dvl1 = (l1 / (y + s)) * ((m / (1 + y)) * ((1 - s) / (1 + y)) ** m) - (
        l1 / (y + s) ** 2
    ) * (1 - ((1 - s) / (1 + y)) ** m)
    dvl2 = (l2 / (y - (1 + r) * (1 - s) + 1)) * (
        (m / (1 + y)) * ((1 + r) * (1 - s) / (1 + y)) ** m
    ) - (l2 / (y - (1 + r) * (1 - s) + 1) ** 2) * (
        1 - ((1 + r) * (1 - s) / (1 + y)) ** m
    )
    dvb = -m * l3 / (1 + y) ** (m + 1)
    adjfactor = 12 * (1 + beyield / 200) ** (5 / 6)
    staticdv01 = -(dvl1 + dvl2 + dvb) / adjfactor  # adjustment factor to reflect
    # derivative based on bond equivalent yield (using the chain rule)
    moddurn = staticdv01 / (price / 100)

    # WAL
    l1 = (r * (1 - s) + s * (1 + r)) / qn  # HACK: lambda 1 assuming no servicing fee;
    # price/moddurn fine withou the servicing fee adjustment, but the WAL math breaks
    l2 = (s * (1 + r)) * (1 - 1 / qn)  # HACK: lambda 2 assuming no servicing fee;
    # price/moddurn fine withou the servicing fee adjustment, but the WAL math breaks
    multl1 = l1 - r / qn
    wall1 = (
        (1 - ((1 - s)) ** (m + 1)) / (1 - ((1 - s))) - (m + 1) * ((1 - s)) ** m
    ) / (1 - ((1 - s)))
    multl2 = l2 + (r / qn) / (1 + r) ** n
    wall2 = (
        (1 - ((1 - s) * (1 + r)) ** (m + 1)) / (1 - ((1 - s) * (1 + r)))
        - (m + 1) * ((1 - s) * (1 + r)) ** m
    ) / (1 - ((1 - s) * (1 + r)))

    qnlessm = 1 - 1 / (1 + r) ** (n - m)
    walb = m * qnlessm / qn * (1 - s) ** (m)

    wal = (multl1 * wall1 + multl2 * wall2 + walb) / 12  # wal1+wal2+walb

    # day and delay adjustments
    forwardprice = price * (1 + y) ** ((dayofmonth - 1 + 30 - delay) / 30)
    cleanprice = forwardprice - coupon * (dayofmonth - 1) / 360

    waladjustment = (dayofmonth - 1 + 30 - delay) / 360
    adjwal = wal - waladjustment
    adjmoddurn = moddurn - waladjustment / (1 + beyield / 200)

    if root is None:
        return (cleanprice, adjmoddurn, adjwal)
    else:
        return cleanprice - root


# def bc35_price_gen(self, beyield='beyield', coupon='coupon', cpr='cpr',
def mtgprice(
    # self,
    beyield="beyield",
    coupon="coupon",
    cpr="cpr",
    age="age",
    origfixedterm="origfixedterm",
    origterm="origterm",
    dayofmonth=1,
    delay=47,
    servicing_fee="servicing_fee",
    price="price",
    moodurn="moddurn",
    wal="wal",
    price_only=True,
):
    self = current._df

    cleanprice, adjmoddurn, adjwal = _bc35_price(
        beyield=self[beyield],
        coupon=self[coupon],
        cpr=self[cpr],
        age=self[age],
        origfixedterm=self[origfixedterm],
        origterm=self[origterm],
        dayofmonth=dayofmonth,
        delay=delay,
        servicing_fee=self[servicing_fee],
    )
    # df = self.copy()
    self[price] = pd.Series(cleanprice)
    if not price_only:
        self[moodurn] = pd.Series(adjmoddurn)
        self[wal] = pd.Series(adjwal)
    # return df
    _print(current.df)


'''
def _pdbc35(
    _yield, dayofmonth=1, delay=47, forsolver=True
):  # 47 day delay is mkt standard per JPM & GS
    """
    This implements BC35 for a pandas DataFrame on a loan level basis

    expects to find rate, origfixedterm, age, servicing_fee,
    """
    # epsilon = tmp.servicing_fee/1200

    tmp["r"] = tmp.rate / 1200
    tmp["y"] = (1 + _yield / 200) ** (1 / 6) - 1
    tmp["s"] = 1 - (1 - tmp._cpr / 100) ** (1 / 12)

    tmp["m"] = tmp.origfixedterm - tmp.age
    tmp["n"] = tmp.origterm - tmp.age

    tmp["qn"] = 1 - 1 / (1 + tmp.r) ** tmp.n
    tmp["l1"] = (
        tmp.r * (1 - tmp.s) - tmp.servicing_fee / 1200 + tmp.s * (1 + tmp.r)
    ) / tmp.qn  # lambda 1
    tmp["l2"] = (tmp.s * (1 + tmp.r) - tmp.servicing_fee / 1200) * (
        1 - 1 / tmp.qn
    )  # lambda 2
    tmp["l3"] = (
        (1 - 1 / tmp.qn) * ((1 + tmp.r) * (1 - tmp.s)) ** tmp.m
        + (1 / tmp.qn) * (1 - tmp.s) ** tmp.m
    )  # lambda 3

    tmp["pvl1"] = (tmp.l1 / (tmp.y + tmp.s)) * (
        1 - ((1 - tmp.s) / (1 + tmp.y)) ** tmp.m
    )  # PV of lamda 1 term
    tmp["pvl2"] = (tmp.l2 / (tmp.y - (1 + tmp.r) * (1 - tmp.s) + 1)) * (
        1 - ((1 + tmp.r) * (1 - tmp.s) / (1 + tmp.y)) ** tmp.m
    )  # PV of lamda 2 term
    tmp["pvb"] = tmp.l3 / (1 + tmp.y) ** tmp.m  # PV of the balloon term
    tmp["price"] = 100 * (tmp.pvl1 + tmp.pvl2 + tmp.pvb)

    tmp["forwardprice"] = tmp.price * (1 + tmp.y) ** (
        (dayofmonth - 1 + 30 - delay) / 30
    )
    tmp["cleanprice"] = tmp.forwardprice - tmp.rate * (dayofmonth - 1) / 360

    if forsolver == True:
        return (
            tmp["cleanprice"] - tmp["_price"]
        )  # (round(cleanprice,6), round(adjmoddurn,2), round(adjwal,2))
    else:
        # Mod Durn
        tmp["dvl1"] = (tmp.l1 / (tmp.y + tmp.s)) * (
            (tmp.m / (1 + tmp.y)) * ((1 - tmp.s) / (1 + tmp.y)) ** tmp.m
        ) - (tmp.l1 / (tmp.y + tmp.s) ** 2) * (1 - ((1 - tmp.s) / (1 + tmp.y)) ** tmp.m)
        tmp["dvl2"] = (tmp.l2 / (tmp.y - (1 + tmp.r) * (1 - tmp.s) + 1)) * (
            (tmp.m / (1 + tmp.y)) * ((1 + tmp.r) * (1 - tmp.s) / (1 + tmp.y)) ** tmp.m
        ) - (tmp.l2 / (tmp.y - (1 + tmp.r) * (1 - tmp.s) + 1) ** 2) * (
            1 - ((1 + tmp.r) * (1 - tmp.s) / (1 + tmp.y)) ** tmp.m
        )
        tmp["dvb"] = -tmp.m * tmp.l3 / (1 + tmp.y) ** (tmp.m + 1)
        tmp["adjfactor"] = 12 * (1 + _yield / 200) ** (5 / 6)
        tmp["staticdv01"] = (
            -(tmp.dvl1 + tmp.dvl2 + tmp.dvb) / tmp.adjfactor
        )  # adjustment factor to reflect derivative based on bond equivalent yield (using the chain rule)
        tmp["moddurn"] = tmp.staticdv01 / (tmp.price / 100)

        # WAL
        tmp["l1"] = (
            tmp.r * (1 - tmp.s) + tmp.s * (1 + tmp.r)
        ) / tmp.qn  # HACK: lambda 1 assuming no servicing fee; price/moddurn fine withou the servicing fee adjustment, but the WAL math breaks
        tmp["l2"] = (tmp.s * (1 + tmp.r)) * (
            1 - 1 / tmp.qn
        )  # HACK: lambda 2 assuming no servicing fee; price/moddurn fine withou the servicing fee adjustment, but the WAL math breaks
        tmp["multl1"] = tmp.l1 - tmp.r / tmp.qn
        tmp["wall1"] = (
            (1 - ((1 - tmp.s)) ** (tmp.m + 1)) / (1 - ((1 - tmp.s)))
            - (tmp.m + 1) * ((1 - tmp.s)) ** tmp.m
        ) / (1 - ((1 - tmp.s)))
        tmp["multl2"] = tmp.l2 + (tmp.r / tmp.qn) / (1 + tmp.r) ** tmp.n
        tmp["wall2"] = (
            (1 - ((1 - tmp.s) * (1 + tmp.r)) ** (tmp.m + 1))
            / (1 - ((1 - tmp.s) * (1 + tmp.r)))
            - (tmp.m + 1) * ((1 - tmp.s) * (1 + tmp.r)) ** tmp.m
        ) / (1 - ((1 - tmp.s) * (1 + tmp.r)))

        tmp["qnlessm"] = 1 - 1 / (1 + tmp.r) ** (tmp.n - tmp.m)
        tmp["walb"] = tmp.m * tmp.qnlessm / tmp.qn * (1 - tmp.s) ** (tmp.m)

        tmp["wal"] = (
            tmp.multl1 * tmp.wall1 + tmp.multl2 * tmp.wall2 + tmp.walb
        ) / 12  # wal1+wal2+walb

        tmp["waladjustment"] = (dayofmonth - 1 + 30 - delay) / 360
        tmp["_wal"] = tmp.wal - tmp.waladjustment
        tmp["_moddurn"] = tmp.moddurn - tmp.waladjustment / (1 + _yield / 200)

        tmp["_price"] = tmp["cleanprice"]
        return [tmp._price, tmp._moddurn, tmp._wal]


def pricegen():
    self = current._df

    global tmp  # globalized so it can be run by the pdbc35 program
    tmp = self.copy()
    if "servicing_fee" not in tmp:
        tmp["servicing_fee"] = 0
    computedprice = _pdbc35(_yield=tmp["_yield"], forsolver=False)  # + all['mtmprice']
    pandasprice = pd.DataFrame(computedprice)
    pandasprice = pandasprice.transpose()
    del tmp
    # newdf = deepcopy(self)
    self["_price"] = pandasprice._price
    self["_wal"] = pandasprice._wal
    self["_moddurn"] = pandasprice._moddurn
    # return newdf


def yieldgen(self):
    from scipy import optimize

    self = current._df

    # Note: this is very prone to producing bad results if/when field have missing values
    global tmp  # globalized so it can be run be the pdbc35 program
    tmp = copy(self)
    if "servicing_fee" not in tmp:
        tmp["servicing_fee"] = 0
    sol = optimize.root(
        _pdbc35, self["rate"] - 0.50, method="df-sane", tol=0.00001
    )  # initial guess = yield is 50bp under the note rate
    del tmp
    # newdf = deepcopy(self)
    self["_yield"] = sol.x  # scipyyield._yield
    # self["temp_price"] = self._price
    # self = self.pricegen()  # run pricegen to get wal and moddurn
    # newdf[
    #     "_price"
    # ] = (
    #     newdf.temp_price
    # )  # It a problem when pivoting yield table if prices aren't identical
    # del newdf["temp_price"]
    # return newdf


mtgprice = pricegen


# def _bc35_price(
#     beyield,
#     coupon,
#     cpr,
#     age,
#     origfixedterm,
#     origterm,
#     dayofmonth=1,
#     delay=47,
#     servicing_fee=0,
#     root=None,
# ):  # 47 is market standard according to Goldmach Sachs
#     """This implements BC35 in core Python (vs columnwise in a pandas.DataFrame)
#     """
#     epsilon = servicing_fee / 1200

#     r = coupon / 1200
#     y = (1 + beyield / 200) ** (1 / 6) - 1
#     s = 1 - (1 - cpr / 100) ** (1 / 12)

#     m = origfixedterm - age
#     n = origterm - age

#     qn = 1 - 1 / (1 + r) ** n
#     l1 = (r * (1 - s) - epsilon + s * (1 + r)) / qn  # lambda 1
#     l2 = (s * (1 + r) - epsilon) * (1 - 1 / qn)  # lambda 2
#     l3 = (1 - 1 / qn) * ((1 + r) * (1 - s)) ** m + (1 / qn) * (1 - s) ** m  # lambda 3

#     pvl1 = (l1 / (y + s)) * (1 - ((1 - s) / (1 + y)) ** m)  # PV of lamda 1 term
#     pvl2 = (l2 / (y - (1 + r) * (1 - s) + 1)) * (
#         1 - ((1 + r) * (1 - s) / (1 + y)) ** m
#     )  # PV of lamda 2 term
#     pvb = l3 / (1 + y) ** m  # PV of the balloon term
#     price = 100 * (pvl1 + pvl2 + pvb)

#     # Mod Durn
#     dvl1 = (l1 / (y + s)) * ((m / (1 + y)) * ((1 - s) / (1 + y)) ** m) - (
#         l1 / (y + s) ** 2
#     ) * (1 - ((1 - s) / (1 + y)) ** m)
#     dvl2 = (l2 / (y - (1 + r) * (1 - s) + 1)) * (
#         (m / (1 + y)) * ((1 + r) * (1 - s) / (1 + y)) ** m
#     ) - (l2 / (y - (1 + r) * (1 - s) + 1) ** 2) * (
#         1 - ((1 + r) * (1 - s) / (1 + y)) ** m
#     )
#     dvb = -m * l3 / (1 + y) ** (m + 1)
#     adjfactor = 12 * (1 + beyield / 200) ** (5 / 6)
#     staticdv01 = -(dvl1 + dvl2 + dvb) / adjfactor  # adjustment factor to reflect
#     # derivative based on bond equivalent yield (using the chain rule)
#     moddurn = staticdv01 / (price / 100)

#     # WAL
#     l1 = (r * (1 - s) + s * (1 + r)) / qn  # HACK: lambda 1 assuming no servicing fee;
#     # price/moddurn fine withou the servicing fee adjustment, but the WAL math breaks
#     l2 = (s * (1 + r)) * (1 - 1 / qn)  # HACK: lambda 2 assuming no servicing fee;
#     # price/moddurn fine withou the servicing fee adjustment, but the WAL math breaks
#     multl1 = l1 - r / qn
#     wall1 = (
#         (1 - ((1 - s)) ** (m + 1)) / (1 - ((1 - s))) - (m + 1) * ((1 - s)) ** m
#     ) / (1 - ((1 - s)))
#     multl2 = l2 + (r / qn) / (1 + r) ** n
#     wall2 = (
#         (1 - ((1 - s) * (1 + r)) ** (m + 1)) / (1 - ((1 - s) * (1 + r)))
#         - (m + 1) * ((1 - s) * (1 + r)) ** m
#     ) / (1 - ((1 - s) * (1 + r)))

#     qnlessm = 1 - 1 / (1 + r) ** (n - m)
#     walb = m * qnlessm / qn * (1 - s) ** (m)

#     wal = (multl1 * wall1 + multl2 * wall2 + walb) / 12  # wal1+wal2+walb

#     # day and delay adjustments
#     forwardprice = price * (1 + y) ** ((dayofmonth - 1 + 30 - delay) / 30)
#     cleanprice = forwardprice - coupon * (dayofmonth - 1) / 360

#     waladjustment = (dayofmonth - 1 + 30 - delay) / 360
#     adjwal = wal - waladjustment
#     adjmoddurn = moddurn - waladjustment / (1 + beyield / 200)

#     if root is None:
#         return (cleanprice, adjmoddurn, adjwal)
#     else:
#         return cleanprice - root


# # def bc35_price_gen(
# def mtgprice(
#     beyield="beyield",
#     coupon="coupon",
#     cpr="cpr",
#     age="age",
#     origfixedterm="origfixedterm",
#     origterm="origterm",
#     dayofmonth=1,
#     delay=47,
#     servicing_fee="servicing_fee",
#     price="price",
#     moodurn="moddurn",
#     wal="wal",
# ):
#     self = current._df

#     cleanprice, adjmoddurn, adjwal = _bc35_price(
#         beyield=self[beyield],
#         coupon=self[coupon],
#         cpr=self[cpr],
#         age=self[age],
#         origfixedterm=self[origfixedterm],
#         origterm=self[origterm],
#         dayofmonth=dayofmonth,
#         delay=delay,
#         servicing_fee=self[servicing_fee],
#     )
#     # df = self.copy()
#     self[price] = pd.Series(cleanprice)
#     self[moodurn] = pd.Series(adjmoddurn)
#     self[wal] = pd.Series(adjwal)
#     # return df
'''

pdexplorer/finance/mtgprice.py
pandichef/pdexplorer
# UNDER CONSTRUCTION
# BC35 calculator: price -> yield
# https://github.com/pandichef/magicpandas/blob/master/magicpandas/frame.py#L496
# https://github.com/pandichef/pandicake/blob/master/pandicake/pdmonkey/yieldgen.py
import pandas as pd
from .._dataset import current
from .mtgprice import _bc35_price
from .._print import _print


def _bc35_yield(
    price,
    coupon,
    cpr,
    age,
    origfixedterm,
    origterm,
    dayofmonth=1,
    delay=47,
    servicing_fee=0,
):

    from scipy import optimize

    # 47 is the market standard according to Goldman Sachs' loan trader
    result = optimize.root(
        _bc35_price,
        coupon,
        args=(
            coupon,
            cpr,
            age,
            origfixedterm,
            origterm,
            dayofmonth,
            delay,
            servicing_fee,
            price,
        ),
        method="df-sane",
    )

    try:
        # if result.x is a scalar
        # if result.x is np.array or pd.Series, then float returns a TypeError
        return float(result.x)
    except TypeError:
        return result.x


def mtgyield(
    # self,
    price="price",
    coupon="coupon",
    cpr="cpr",
    age="age",
    origfixedterm="origfixedterm",
    origterm="origterm",
    dayofmonth=1,
    delay=47,
    servicing_fee="servicing_fee",
    beyield="beyield",
):
    self = current._df

    beyield_array = _bc35_yield(
        price=self[price],
        coupon=self[coupon],
        cpr=self[cpr],
        age=self[age],
        origfixedterm=self[origfixedterm],
        origterm=self[origterm],
        dayofmonth=dayofmonth,
        delay=delay,
        servicing_fee=self[servicing_fee],  # type: ignore
    )
    # df = self.copy()
    self[beyield] = pd.Series(beyield_array)
    # return df
    _print(current.df)


# from .mtgprice import yieldgen as mtgyield

# mtgyield = None

# from .mtgprice import _bc35_price


# def _bc35_yield(
#     price,
#     coupon,
#     cpr,
#     age,
#     origfixedterm,
#     origterm,
#     dayofmonth=1,
#     delay=47,
#     servicing_fee=0,
# ):
#     from scipy import optimize

#     # 47 is the market standard according to Goldman Sachs' loan trader
#     result = optimize.root(
#         _bc35_price,
#         coupon,
#         args=(
#             coupon,
#             cpr,
#             age,
#             origfixedterm,
#             origterm,
#             dayofmonth,
#             delay,
#             servicing_fee,
#             price,
#         ),
#         method="df-sane",
#     )

#     try:
#         # if result.x is a scalar
#         # if result.x is np.array or pd.Series, then float returns a TypeError
#         return float(result.x)
#     except TypeError:
#         return result.x


# def bc35_yield_gen(
#     self,
#     price="price",
#     coupon="coupon",
#     cpr="cpr",
#     age="age",
#     origfixedterm="origfixedterm",
#     origterm="origterm",
#     dayofmonth=1,
#     delay=47,
#     servicing_fee="servicing_fee",
#     beyield="beyield",
# ):
#     self = current._df

#     beyield_array = _bc35_yield(
#         price=self[price],
#         coupon=self[coupon],
#         cpr=self[cpr],
#         age=self[age],
#         origfixedterm=self[origfixedterm],
#         origterm=self[origterm],
#         dayofmonth=dayofmonth,
#         delay=delay,
#         servicing_fee=self[servicing_fee],
#     )
#     # df = self.copy()
#     self[beyield] = pd.Series(beyield_array)
#     # return df

pdexplorer/finance/mtgyield.py
pandichef/pdexplorer
# https://colab.research.google.com/drive/1FjTsqbYKphl9kL-eILgUc-bl4zVThL8F?usp=sharing
# https://huggingface.co/blog/fine-tune-wav2vec2-english
import uuid
import numpy as np
from pdexplorer._print import _print
from pdexplorer._dataset import current
from pdexplorer._commandarg import parse


def audio_classification(
    # df: pd.DataFrame,
    commandarg: str,
    # model_name: str = "distilbert-base-uncased",
    model_name: str,
    test_size: float = 0.3,
    sampling_rate=96000,
) -> None:
    """https://huggingface.co/docs/transformers/tasks/audio_classification"""
    import torch
    import datasets
    import evaluate
    from transformers import (
        AutoFeatureExtractor,
        AutoModelForSequenceClassification,
        AutoTokenizer,
        AutoModelForAudioClassification,
        DataCollatorWithPadding,
        TrainingArguments,
        Trainer,
    )

    _ = parse(commandarg, "varlist")
    assert len(_["varlist"].split()) == 2  # label and text columns only #
    label_var = _["varlist"].split()[0]  # assumed to be "label" #
    audio_var = _["varlist"].split()[1]  # assumed to be "audio" #

    df = current.df.rename(columns={label_var: "label", audio_var: "audio"})
    columns_to_drop = list(df.columns)
    columns_to_drop.remove("label")
    columns_to_drop.remove("audio")
    df = df.drop(columns_to_drop, axis=1)
    ds = datasets.Dataset.from_pandas(df)
    ds = ds.train_test_split(test_size=test_size)

    # tokenizer = AutoTokenizer.from_pretrained(model_name)
    feature_extractor = AutoFeatureExtractor.from_pretrained(model_name)
    ds = ds.cast_column("audio", datasets.Audio(sampling_rate=sampling_rate))

    def preprocess_function(examples):
        audio_arrays = [x["array"] for x in examples["audio"]]
        inputs = feature_extractor(
            audio_arrays,
            sampling_rate=feature_extractor.sampling_rate,
            max_length=sampling_rate,
            truncation=True,
        )
        return inputs

    encoded_ds = ds.map(preprocess_function, remove_columns="audio", batched=True)
    accuracy = evaluate.load("accuracy")

    def compute_metrics(eval_pred):
        predictions = np.argmax(eval_pred.predictions, axis=1)
        return accuracy.compute(predictions=predictions, references=eval_pred.label_ids)

    model = AutoModelForAudioClassification.from_pretrained(
        model_name, num_labels=df["label"].nunique()
    )

    output_dir = f"ft-{model_name.replace('/','_')}-{str(uuid.uuid4())[:8]}"
    training_args = TrainingArguments(
        output_dir=output_dir,
        evaluation_strategy="epoch",
        save_strategy="epoch",
        learning_rate=3e-5,
        per_device_train_batch_size=32,
        gradient_accumulation_steps=4,
        per_device_eval_batch_size=32,
        num_train_epochs=10,
        warmup_ratio=0.1,
        logging_steps=10,
        load_best_model_at_end=True,
        metric_for_best_model="accuracy",
        # push_to_hub=True,
        use_cpu=not torch.cuda.is_available(),
        report_to="none",  # prevent input() #
    )

    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=encoded_ds["train"],
        eval_dataset=encoded_ds["test"],
        tokenizer=feature_extractor,
        compute_metrics=compute_metrics,
    )

    trainer.train()
    trainer.save_model()  # saved to output_dir
    _print(f"Saved model to ./{output_dir}")
    torch.cuda.empty_cache()
    current.last_huggingface_ftmodel_dir = output_dir


# print(_sentiment_analysis("stars text"))

pdexplorer/fine_tuning/_audio_classification.py
pandichef/pdexplorer
import uuid
import numpy as np
from pdexplorer._print import _print
from pdexplorer._dataset import current
from pdexplorer._commandarg import parse


# from pdexplorer.tests.fixtures import eli5

# import pandas as pd

# df = pd.DataFrame.from_records(eli5)


def fill_mask(
    # df: pd.DataFrame,
    commandarg: str,
    # model_name: str = "distilroberta-base",
    model_name: str,
    test_size: float = 0.3,
    block_size: int = 128,
):
    """
    https://huggingface.co/docs/transformers/tasks/masked_language_modeling
    """
    import torch
    import datasets
    from transformers import (
        AutoModelForMaskedLM,
        AutoTokenizer,
        DataCollatorForLanguageModeling,
        TrainingArguments,
        Trainer,
    )

    _ = parse(commandarg, "varlist")
    assert len(_["varlist"].split()) == 1  # text column only #
    text_var = _["varlist"].split()[0]  # assumed to be "text" #

    df = current.df.rename(columns={text_var: "text"})

    # task = "sentiment-analysis"
    ds = datasets.Dataset.from_pandas(df)
    ds = ds.train_test_split(test_size=test_size)
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    ds = ds.flatten()

    def preprocess_function(examples):
        from transformers import AutoTokenizer

        tokenizer = AutoTokenizer.from_pretrained(model_name)
        return tokenizer([" ".join(x) for x in examples["text"]])

    tokenized_ds = ds.map(
        preprocess_function,
        batched=True,
        num_proc=4,
        remove_columns=ds["train"].column_names,
    )

    def group_texts(examples):
        # Concatenate all texts.
        concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
        total_length = len(concatenated_examples[list(examples.keys())[0]])
        # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
        # customize this part to your needs.
        if total_length >= block_size:
            total_length = (total_length // block_size) * block_size
        # Split by chunks of block_size.
        result = {
            k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
            for k, t in concatenated_examples.items()
        }
        return result

    lm_dataset = tokenized_ds.map(group_texts, batched=True, num_proc=4)

    # Documentation pads this way: #
    # tokenizer.pad_token = tokenizer.eos_token #
    # This worked in a script, but not within a function call #
    tokenizer.add_special_tokens({"pad_token": "[PAD]"})

    data_collator = DataCollatorForLanguageModeling(
        tokenizer=tokenizer, mlm_probability=0.15,
    )

    model = AutoModelForMaskedLM.from_pretrained(
        model_name, torch_dtype=torch.bfloat16 if current.use_torch_bfloat16 else "auto"
    )
    print(f"""Uses {model.get_memory_footprint()/1073741824} GB.""")

    output_dir = f"ft-{model_name}-{str(uuid.uuid4())[:8]}"

    training_args = TrainingArguments(
        output_dir=output_dir,
        evaluation_strategy="epoch",
        learning_rate=2e-5,
        num_train_epochs=3,
        weight_decay=0.01,
        # push_to_hub=True,
        use_cpu=not torch.cuda.is_available(),
        report_to="none",  # prevent input() #
    )

    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=lm_dataset["train"],
        eval_dataset=lm_dataset["test"],
        data_collator=data_collator,
    )

    trainer.train()
    trainer.save_model()  # saved to output_dir
    _print(f"Saved model to ./{output_dir}")
    torch.cuda.empty_cache()
    current.last_huggingface_ftmodel_dir = output_dir

pdexplorer/fine_tuning/_fill_mask.py
pandichef/pdexplorer
import uuid
import numpy as np

# import pandas as pd

# from pdexplorer.tests.fixtures import yelp_reviews
from pdexplorer._print import _print
from pdexplorer._dataset import current
from pdexplorer._commandarg import parse

# df = pd.DataFrame.from_records(yelp_reviews)


def sentiment_analysis(
    # df: pd.DataFrame,
    commandarg: str,
    # model_name: str = "distilbert-base-uncased",
    model_name: str,
    test_size: float = 0.3,
) -> None:
    """
    https://huggingface.co/docs/transformers/tasks/sequence_classification
    """
    import torch
    import datasets
    import evaluate
    from transformers import (
        AutoModelForSequenceClassification,
        AutoTokenizer,
        DataCollatorWithPadding,
        TrainingArguments,
        Trainer,
    )

    _ = parse(commandarg, "varlist")
    assert len(_["varlist"].split()) == 2  # label and text columns only #
    label_var = _["varlist"].split()[0]  # assumed to be "label" #
    text_var = _["varlist"].split()[1]  # assumed to be "text" #

    df = current.df.rename(columns={label_var: "label", text_var: "text"})

    # task = "sentiment-analysis"
    ds = datasets.Dataset.from_pandas(df)
    ds = ds.train_test_split(test_size=test_size)
    tokenizer = AutoTokenizer.from_pretrained(model_name)

    def preprocess_function(examples):
        return tokenizer(examples["text"], truncation=True)

    tokenized_ds = ds.map(preprocess_function, batched=True)
    data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
    accuracy = evaluate.load("accuracy")

    def compute_metrics(eval_pred):
        predictions, labels = eval_pred
        predictions = np.argmax(predictions, axis=1)
        return accuracy.compute(predictions=predictions, references=labels)

    model = AutoModelForSequenceClassification.from_pretrained(
        model_name,
        num_labels=df["label"].nunique(),
        torch_dtype=torch.bfloat16 if current.use_torch_bfloat16 else "auto",
    )
    print(f"""Uses {model.get_memory_footprint()/1073741824} GB.""")

    output_dir = f"ft-{model_name}-{str(uuid.uuid4())[:8]}"
    trainer = Trainer(
        model=model,
        args=TrainingArguments(
            output_dir=output_dir,
            learning_rate=2e-5,
            per_device_train_batch_size=16,
            per_device_eval_batch_size=16,
            num_train_epochs=2,
            weight_decay=0.01,
            evaluation_strategy="epoch",
            save_strategy="epoch",
            load_best_model_at_end=True,
            # push_to_hub=True,
            use_cpu=not torch.cuda.is_available(),
            report_to="none",  # prevent input() #
        ),
        train_dataset=tokenized_ds["train"],
        eval_dataset=tokenized_ds["test"],
        tokenizer=tokenizer,
        data_collator=data_collator,
        compute_metrics=compute_metrics,
    )

    trainer.train()
    trainer.save_model()  # saved to output_dir
    _print(f"Saved model to ./{output_dir}")
    torch.cuda.empty_cache()
    current.last_huggingface_ftmodel_dir = output_dir


# print(_sentiment_analysis("stars text"))

pdexplorer/fine_tuning/_sentiment_analysis.py
pandichef/pdexplorer
import uuid
import numpy as np
from pdexplorer._print import _print
from pdexplorer._dataset import current
from pdexplorer._commandarg import parse


def text_generation(
    # df: pd.DataFrame,
    commandarg: str,
    # model_name: str = "distilgpt2",
    model_name: str,
    test_size: float = 0.3,
    block_size: int = 128,
) -> None:
    """
    https://huggingface.co/docs/transformers/tasks/language_modeling
    """
    import torch
    import datasets
    from transformers import (
        AutoModelForCausalLM,
        AutoTokenizer,
        DataCollatorForLanguageModeling,
        TrainingArguments,
        Trainer,
    )

    _ = parse(commandarg, "varlist")
    assert len(_["varlist"].split()) == 1  # text column only #
    text_var = _["varlist"].split()[0]  # assumed to be "text" #

    df = current.df.rename(columns={text_var: "text"})

    # task = "sentiment-analysis"
    ds = datasets.Dataset.from_pandas(df)
    ds = ds.train_test_split(test_size=test_size)
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    ds = ds.flatten()

    def preprocess_function(examples):
        from transformers import AutoTokenizer

        tokenizer = AutoTokenizer.from_pretrained(model_name)
        return tokenizer([" ".join(x) for x in examples["text"]])

    tokenized_ds = ds.map(
        preprocess_function,
        batched=True,
        num_proc=4,
        remove_columns=ds["train"].column_names,
    )

    def group_texts(examples):
        # Concatenate all texts.
        concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
        total_length = len(concatenated_examples[list(examples.keys())[0]])
        # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
        # customize this part to your needs.
        if total_length >= block_size:
            total_length = (total_length // block_size) * block_size
        # Split by chunks of block_size.
        result = {
            k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
            for k, t in concatenated_examples.items()
        }
        return result

    lm_dataset = tokenized_ds.map(group_texts, batched=True, num_proc=4)

    # Documentation pads this way: #
    # tokenizer.pad_token = tokenizer.eos_token #
    # This worked in a script, but not within a function call #
    tokenizer.add_special_tokens({"pad_token": "[PAD]"})

    # data_collator = DataCollatorForLanguageModeling(
    #     tokenizer=tokenizer, mlm_probability=0.15, mlm=False
    # )
    data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)

    model = AutoModelForCausalLM.from_pretrained(
        model_name, torch_dtype=torch.bfloat16 if current.use_torch_bfloat16 else "auto"
    )
    print(f"""Uses {model.get_memory_footprint()/1073741824} GB.""")

    output_dir = f"ft-{model_name}-{str(uuid.uuid4())[:8]}"

    training_args = TrainingArguments(
        output_dir=output_dir,
        evaluation_strategy="epoch",
        learning_rate=2e-5,
        num_train_epochs=3,
        weight_decay=0.01,
        # push_to_hub=True,
        use_cpu=not torch.cuda.is_available(),
        report_to="none",  # prevent input() #
    )

    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=lm_dataset["train"],
        eval_dataset=lm_dataset["test"],
        data_collator=data_collator,
    )

    trainer.train()
    trainer.save_model()  # saved to output_dir
    _print(f"Saved model to ./{output_dir}")
    torch.cuda.empty_cache()
    current.last_huggingface_ftmodel_dir = output_dir

pdexplorer/fine_tuning/_text_generation.py
pandichef/pdexplorer
import time
import os
import io
import json
import pandas as pd
from .._dataset import current
from .._commandarg import parse
from .._print import _print


def _convert_tabular_format_to_openai_format(df) -> list:
    # Convert DataFrame to dictionary with 'records' orientation
    df_dict = df.to_dict(orient="records")

    # Use map to transform each record into the desired format
    records = list(
        map(
            lambda record: {
                "messages": [
                    {"role": role, "content": content}
                    for role, content in record.items()
                ]
            },
            df_dict,
        )
    )

    return records


def _write_records_to_stringio(records: list) -> io.StringIO:
    output = io.StringIO()
    for record in records:
        output.write(json.dumps(record))
        output.write("\n")
    output.seek(0)
    return output


class FineTuningJobHelper:
    def __init__(self, openai_ftjob_id: str) -> None:
        self.openai_ftjob_id = openai_ftjob_id

    def retrieve(self):
        import openai

        return openai.FineTuningJob.retrieve(self.openai_ftjob_id)

    def cancel(self):
        import openai

        return openai.FineTuningJob.cancel(self.openai_ftjob_id)

    def list_events(self, limit=10):
        import openai

        return openai.FineTuningJob.list_events(  # type: ignore
            id=self.openai_ftjob_id, limit=limit
        )

    def delete_model(self):
        import openai

        model_name = self.retrieve().to_dict()["fine_tuned_model"]
        return openai.Model.delete(model_name)

    def list_jobs(self):
        import openai

        jobs = pd.DataFrame(openai.FineTuningJob.list()["data"])  # type: ignore
        jobs["created_at"] = pd.to_datetime(jobs["created_at"], unit="s")
        jobs["finished_at"] = pd.to_datetime(jobs["finished_at"], unit="s")
        jobs = jobs.sort_values("created_at", ascending=False)
        return jobs

    def list_models(self):
        import openai

        models = pd.DataFrame(openai.Model.list()["data"])  # type: ignore
        models = models[models["id"].str.contains("personal")]
        models.reset_index(inplace=True, drop=True)
        models["created"] = pd.to_datetime(models["created"], unit="s")
        return models


def ftgpt(
    commandarg: str, model_name: str = "gpt-3.5-turbo", nonblocking=True
) -> FineTuningJobHelper:
    """
    1. creates a training file for OpenAI Fine-tuning API
    2. Upload a training file; https://platform.openai.com/docs/guides/fine-tuning/upload-a-training-file
    3. Create a fine-tuned model; https://platform.openai.com/docs/guides/fine-tuning/create-a-fine-tuned-model

    """
    import openai

    openai.api_key = os.getenv("OPENAI_API_KEY")

    _ = parse(commandarg, "varlist")
    assert len(_["varlist"].split()) == 3
    assistant_var = _["varlist"].split()[0]
    user_var = _["varlist"].split()[1]
    system_var = _["varlist"].split()[2]

    df = current.df.rename(
        columns={assistant_var: "assistant", user_var: "user", system_var: "system"}
    )

    records = _convert_tabular_format_to_openai_format(df)
    stringio_object = _write_records_to_stringio(records)
    openai_file_object = openai.File.create(file=stringio_object, purpose="fine-tune")
    openai_file_id = openai_file_object.to_dict()["id"]  # type: ignore
    openai_ftjob_object = openai.FineTuningJob.create(
        training_file=openai_file_id, model=model_name
    )
    openai_ftjob_id = openai_ftjob_object.to_dict()["id"]  # type: ignore
    ftjob_helper = FineTuningJobHelper(openai_ftjob_id)
    current.last_openai_ftjob_id = openai_ftjob_id

    if not nonblocking:
        while ftjob_helper.retrieve().to_dict()["status"] in [
            "validating_files",
            "running",
        ]:
            time.sleep(10)
            print("Waiting for fine-tuning to complete...")

        print("Fine-tuning complete!")
        final_status = ftjob_helper.retrieve().to_dict()["status"]
        print(f"Final status: {final_status}")

    return ftjob_helper


def askgpt(user_content="Hello!", system_content="You are a helpful assistant."):
    import openai
    from openai.error import ServiceUnavailableError

    while True:
        try:
            ftjob_helper = FineTuningJobHelper(current.last_openai_ftjob_id)  # type: ignore
            try:
                model_name = ftjob_helper.retrieve().to_dict()["fine_tuned_model"]
                print(f"(Using fine tuned model {current.last_openai_ftjob_id}.)")
                print()
            except Exception as e:
                model_name = "gpt-3.5-turbo-0613"
                print(
                    f"(Fine tuned model not found.  Using base model {model_name} instead.)"
                )
                print()

            completion = openai.ChatCompletion.create(
                model=model_name,
                messages=[
                    {"role": "system", "content": system_content},
                    {"role": "user", "content": user_content},
                ],
            )
            break
        except ServiceUnavailableError as e:
            pass
            time.sleep(10)

    print(completion.choices[0].message["content"])  # type: ignore

pdexplorer/fine_tuning/ftgpt.py
pandichef/pdexplorer
from .._dataset import current
from .._print import _print
from .._commandarg import parse
from ._sentiment_analysis import sentiment_analysis
from ._fill_mask import fill_mask
from ._text_generation import text_generation
from ._audio_classification import audio_classification

task_map = {  # generated by ChatGPT #
    "sentiment-analysis": {
        "auto_model_class": "AutoModelForSequenceClassification",
        "fine_tuner": sentiment_analysis,
        "default_model": "distilbert-base-uncased",
    },
    "text-classification": {
        "auto_model_class": "AutoModelForSequenceClassification",
        "fine_tuner": sentiment_analysis,
        "default_model": "distilbert-base-uncased",
    },
    "fill-mask": {
        "auto_model_class": "AutoModelForMaskedLM",
        "fine_tuner": fill_mask,
        "default_model": "distilroberta-base",
    },
    "text-generation": {
        "auto_model_class": "AutoModelForCausalLM",
        "fine_tuner": text_generation,
        "default_model": "distilgpt2",
    },
    "audio-classification": {
        "auto_model_class": "AutoModelForAudioClassification",
        "fine_tuner": audio_classification,
        "default_model": "facebook/wav2vec2-base",
        # "default_model": "openai/whisper-tiny",
    },
    "automatic-speech-recognition": {
        "auto_model_class": "AutoModelForAutomaticSpeechRecognition"
    },
    "conversational": {"auto_model_class": "AutoModelForSeq2SeqLM"},
    "depth-estimation": {"auto_model_class": "AutoModelForImageSegmentation"},
    "document-question-answering": {
        "auto_model_class": "AutoModelForQuestionAnswering"
    },
    "feature-extraction": {"auto_model_class": "AutoModel"},
    "image-classification": {"auto_model_class": "AutoModelForImageClassification"},
    "image-segmentation": {"auto_model_class": "AutoModelForImageSegmentation"},
    "image-to-image": {"auto_model_class": "AutoModelForImageSegmentation"},
    "image-to-text": {"auto_model_class": "AutoModelForImageToText"},
    "mask-generation": {"auto_model_class": "AutoModelForImageSegmentation"},
    "ner": {"auto_model_class": "AutoModelForTokenClassification"},
    "object-detection": {"auto_model_class": "AutoModelForObjectDetection"},
    "question-answering": {"auto_model_class": "AutoModelForQuestionAnswering"},
    "summarization": {"auto_model_class": "AutoModelForSeq2SeqLM"},
    "table-question-answering": {"auto_model_class": "AutoModelForQuestionAnswering"},
    "text-to-audio": {"auto_model_class": "AutoModelForTextToSpeech"},
    "text-to-speech": {"auto_model_class": "AutoModelForTextToSpeech"},
    "text2text-generation": {"auto_model_class": "AutoModelForSeq2SeqLM"},
    "token-classification": {"auto_model_class": "AutoModelForTokenClassification"},
    "translation": {"auto_model_class": "AutoModelForSeq2SeqLM"},
    "video-classification": {"auto_model_class": "AutoModelForVideoClassification"},
    "visual-question-answering": {
        "auto_model_class": "AutoModelForImageTextIntegration"
    },
    "vqa": {"auto_model_class": "AutoModelForImageTextIntegration"},
    "zero-shot-audio-classification": {
        "auto_model_class": "AutoModelForZeroShotClassification"
    },
    "zero-shot-classification": {
        "auto_model_class": "AutoModelForZeroShotClassification"
    },
    "zero-shot-image-classification": {
        "auto_model_class": "AutoModelForZeroShotClassification"
    },
    "zero-shot-object-detection": {
        "auto_model_class": "AutoModelForZeroShotObjectDetection"
    },
    "translation_XX_to_YY": {"auto_model_class": "AutoModelForSeq2SeqLM"},
}

# num_columns = {
#     "text-classification": 2,  # text and label
#     "sentiment-analysis": 2,  # text and label
#     "fill-mask": 1,  # text only
# }


# def get_tokenized_data(
#     huggingface_dataset, tokenizer, yvar, xvar=None, task="text-classification"
# ):
#     if task in ["text-classification", "sentiment-analysis"]:

#         def preprocess_function(examples):
#             return tokenizer(examples[xvar], padding="max_length", truncation=True)

#         tokenized_data = huggingface_dataset.map(preprocess_function, batched=True,)
#     elif task in ["fill-mask"]:

#         def preprocess_function(examples):
#             return tokenizer([" ".join(x) for x in examples[yvar]])

#         tokenized_data = huggingface_dataset.map(
#             preprocess_function,
#             batched=True,
#             num_proc=4,
#             remove_columns=huggingface_dataset["train"].column_names,
#         )

#         def group_texts(examples):
#             block_size = 128
#             concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
#             total_length = len(concatenated_examples[list(examples.keys())[0]])
#             if total_length >= block_size:
#                 total_length = (total_length // block_size) * block_size
#             result = {
#                 k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
#                 for k, t in concatenated_examples.items()
#             }
#             return result

#         tokenized_data = tokenized_data.map(group_texts, batched=True, num_proc=4)
#     else:
#         tokenized_data = None
#     return tokenized_data


# default_models = {
#     "sentiment-analysis": "distilbert-base-uncased",
#     "text-classification": "distilbert-base-uncased",
#     "fill-mask": "distilroberta-base",
#     "text-generation": "distilgpt2",
# }


def fthuggingface(
    commandarg: str, task="sentiment-analysis", model_name=None  # varlist #
):
    if model_name is None:
        model_name = task_map[task]["default_model"]
    fine_tuner_function = task_map[task]["fine_tuner"]
    fine_tuner_function(commandarg=commandarg, model_name=model_name)


def askhuggingface(prompt, task="text-classification"):
    import transformers
    from transformers import (
        pipeline,
        AutoTokenizer,
        AutoFeatureExtractor,  # for audio-classification
    )

    _AutoModel = getattr(transformers, task_map[task]["auto_model_class"])
    base_model_name = "-".join(current.last_huggingface_ftmodel_dir.split("-")[1:-1]).replace("_", "/")  # type: ignore

    model = _AutoModel.from_pretrained(current.last_huggingface_ftmodel_dir)
    pipeline_kwargs = {
        "task": task,
        "model": model,
    }

    if task == "audio-classification":
        feature_extractor = AutoFeatureExtractor.from_pretrained(base_model_name)
        pipeline_kwargs.update({"feature_extractor": feature_extractor})
    else:
        tokenizer = AutoTokenizer.from_pretrained(base_model_name)
        pipeline_kwargs.update({"tokenizer": tokenizer})

    fine_tuned_text_classification_pipeline = pipeline(**pipeline_kwargs)
    result = fine_tuned_text_classification_pipeline(prompt)
    # print()  # type: ignore
    return result


##################################################
# def fthuggingface_old(
#     commandarg: str,
#     output_dir="output_dir",
#     num_examples=100,
#     model_name="distilbert-base-uncased",
#     num_labels=5,
#     run_in_sample=False,
# ):
#     # Deprecated #
#     _ = parse(commandarg, "varlist")
#     assert len(_["varlist"].split()) == 2
#     yvar = _["varlist"].split()[0]
#     xvar = _["varlist"].split()[1]
#     # https://huggingface.co/docs/transformers/training#train-with-pytorch-trainer
#     import datasets
#     import torch
#     import numpy as np
#     import evaluate
#     from datasets import load_dataset
#     from transformers import (
#         pipeline,
#         AutoTokenizer,
#         AutoModelForSequenceClassification,  # text-classification
#         AutoModelForMaskedLM,  # fill-mask
#         TrainingArguments,
#         Trainer,
#     )

#     tokenizer = AutoTokenizer.from_pretrained(model_name)

#     dataset = datasets.DatasetDict()
#     dataset["train"] = current.get_huggingface_dataset(split="train")
#     dataset["test"] = current.get_huggingface_dataset(split="test")

#     # tokenize the fine-tuning dataset #
#     def tokenize_function(examples):
#         return tokenizer(examples[xvar], padding="max_length", truncation=True)

#     tokenized_datasets = dataset.map(tokenize_function, batched=True,)

#     # sample from fine-tuning dataset #
#     small_train_dataset = (
#         tokenized_datasets["train"].shuffle(seed=42).select(range(num_examples))
#     )
#     small_eval_dataset = (
#         tokenized_datasets["test"].shuffle(seed=42).select(range(num_examples))
#     )

#     # load model for specific application #
#     # e.g., distilbert-base-uncased loaded for text classification with 5 stars #
#     model = AutoModelForSequenceClassification.from_pretrained(
#         model_name, num_labels=num_labels
#     )

#     if run_in_sample:
#         text_classification_pipeline = pipeline(
#             "text-classification", model=model, tokenizer=tokenizer
#         )

#         def _text_classification_pipeline(x):
#             try:
#                 return int(
#                     text_classification_pipeline(x)[0][yvar].replace("LABEL_", "")
#                 )
#             except RuntimeError:
#                 return -1

#         current._df["_pretrained"] = current._df[xvar].apply(
#             _text_classification_pipeline
#         )

#     # train the model
#     def compute_metrics(eval_pred):
#         logits, labels = eval_pred
#         predictions = np.argmax(logits, axis=-1)
#         return evaluate.load("accuracy").compute(
#             predictions=predictions, references=labels
#         )

#     trainer = Trainer(
#         model=model,
#         args=TrainingArguments(
#             output_dir=output_dir,
#             evaluation_strategy="epoch",
#             no_cuda=not torch.cuda.is_available(),
#             # no_cuda=True,
#             report_to="none"
#             # per_device_train_batch_size=8,
#             # per_gpu_eval_batch_size=8,
#         ),
#         train_dataset=small_train_dataset,
#         eval_dataset=small_eval_dataset,
#         compute_metrics=compute_metrics,
#     )
#     trainer.train()
#     trainer.save_model()  # saved to output_dir
#     _print(f"Saved model to ./{output_dir}")

#     if run_in_sample:
#         fine_tuned_text_classification_pipeline = pipeline(
#             "text-classification",
#             model=AutoModelForSequenceClassification.from_pretrained(output_dir),
#             tokenizer=tokenizer,
#         )

#         def _fine_tuned_text_classification_pipeline(x):
#             try:
#                 return int(
#                     fine_tuned_text_classification_pipeline(x)[0][yvar].replace(
#                         "LABEL_", ""
#                     )
#                 )
#             except RuntimeError:
#                 return -1

#         current._df["_finetuned"] = current._df[xvar].apply(
#             _fine_tuned_text_classification_pipeline
#         )
#         _print(current.df)
#     torch.cuda.empty_cache()

pdexplorer/fine_tuning/fthuggingface.py
pandichef/pdexplorer
# TODO: add in_/if_
from ._dataset import current
from ._print import _print
from ._commandarg import parse


def generate(commandarg: str) -> None:
    _ = parse(commandarg)
    assignment = _["anything"] + "=" + _["="]

    new_column_name = assignment.split("=")[0].strip()
    assert (
        new_column_name not in current.df.columns
    ), 'Column already exists.  See "replace" command.'
    # current.df.eval(assignment, inplace=True)

    if _["if"]:
        current.df.loc[current.df.query(_["if"]).index, _["anything"],] = _["="]
    else:
        current.df.eval(assignment, inplace=True)

    _print(current.df)

pdexplorer/generate.py
pandichef/pdexplorer
# from ._search import search_iterable
from ._dataset import current
from ._print import _print


def gsort(varlist) -> None:
    """Implements Stata's gsort command
    Note Stata's different interpretation of the minus sign here
    Hence search_iterable isn't used

    >>> gsort('origfico -origltv')
    """
    df = current.df  # originally written as a function
    if type(varlist) == str:
        varlist = varlist.split()
    ascending_list = []
    new_varlist = []
    for x in varlist:
        if x[:1] == "-":
            new_varlist.append(x[1:])
            ascending_list.append(False)
        elif x[:1] == "+":
            new_varlist.append(x[1:])
            ascending_list.append(True)
        else:
            new_varlist.append(x)
            ascending_list.append(True)
    df.sort_values(new_varlist, ascending=ascending_list, inplace=True)
    _print(current.df)

pdexplorer/gsort.py
pandichef/pdexplorer
from ._search import search_iterable
from ._dataset import current
from ._print import _print
import pandas as pd
from ._commandarg import parse

# from .keepif import keepif
# from .keepin import keepin
from ._quietly import quietly
from ._dataset import current
from ._search import search_iterable
import pandas as pd
from ._print import _print
from ._stata_slice import _stata_slice
from ._dataset import current
from ._print import _print


def keepin(range: str) -> None:
    range = _stata_slice(range)

    current.df = eval(f"current.df[{range}]")
    _print(current.df)


def keepif(condition: str) -> None:
    current.df.query(condition, inplace=True)
    _print(current.df)


def _keep(df: pd.DataFrame, columns_to_keep: list) -> pd.DataFrame:
    columns_to_drop = [x for x in df.columns if x not in columns_to_keep]
    return df.drop(labels=columns_to_drop, axis=1, inplace=False)


# def keepold(varlist: str) -> None:
#     columns_to_keep = search_iterable(current.df.columns, varlist)
#     current.df = _keep(current.df, columns_to_keep)
#     _print(current.df)


def keep(commandarg: str) -> None:
    _ = parse(commandarg)

    if _["anything"]:
        assert (
            _["in"] is None and _["if"] is None
        ), "drop takes either a varlist or in/if, but not both"
        varlist = search_iterable(current.df.columns, _["anything"])
        current.df = _keep(current._df, varlist)
    elif _["in"] or _["if"]:
        assert (
            _["anything"] is None
        ), "drop takes either a varlist or in/if, but not both"
        with quietly():
            if _["in"]:
                keepin(_["in"])
            if _["if"]:
                keepif(_["if"])
    else:
        raise Exception("drop: Missing Arguments")

    current._df.reset_index(inplace=True, drop=True)
    _print(current._df)

pdexplorer/keep.py
pandichef/pdexplorer
from ._dataset import current


def labeldata(label: str):
    current.metadata["data_label"] = label

pdexplorer/labeldata.py
pandichef/pdexplorer
from ._dataset import current


def labelvariable(varname: str, label: str):
    current.metadata["data_label"][varname] = label

pdexplorer/labelvariable.py
pandichef/pdexplorer
from ._dataset import current
from ._search import search_iterable
import pandas as pd
import warnings
from ._print import _print
from ._stata_slice import _stata_slice

# def list(varlist: str, rows=10):
def lis(varlist=None, if_=None, in_=None):
    """The Stata equivalent is list.  However, lis was used here since list is a 
    built-in type in Python.
    """
    # if isinstance(in_, str) and in_.find("/") > -1:  # Stata's slicing syntax
    #     in_ = in_.replace("/", ":")
    #     _in_list = in_.split(":")
    #     in_ = str(int(_in_list[0]) - 1) + ":" + _in_list[1]
    in_ = _stata_slice(in_)

    df = current.df
    if varlist:
        labels = search_iterable(df.columns, varlist)
        inverted_labels = [x for x in df.columns if x not in labels]
        df_to_display = df.drop(labels=inverted_labels, axis=1, inplace=False)
    else:
        df_to_display = df

    # Note "in_" must come first
    if in_:
        df_to_display = eval(f"df_to_display[{in_}]")

    if if_:
        df_to_display = df_to_display.query(if_, inplace=False)

    if in_:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            pd.set_option("display.max_rows", None)
            # pd.set_option("display.min_rows", None)
            _print(df_to_display)
            pd.reset_option("all")
    else:
        _print(df_to_display)

pdexplorer/lis.py
pandichef/pdexplorer
from ._dataset import current
from ._get_custom_attributes import _get_custom_attributes
from ._patsify import _patsify
from ._print import _print


def logit(varlist: str):
    import statsmodels.formula.api as smf

    df = current.df
    patsy_formula = _patsify(varlist)
    model = smf.logit(patsy_formula, data=df)
    results = model.fit()
    _print(results.summary())
    current.methods, current.properties = _get_custom_attributes(results)
    return results

pdexplorer/logit.py
pandichef/pdexplorer
import os
import pandas as pd
from ._dataset import current
from .use import _use
from .restore import restore
from ._print import _print
from ._search import search_iterable


def merge(
    varlist: str,
    right="preserved",  # by default, use the preserved dataset
    how="left",  # The standard vlookup style merge
    suffixes=(None, "__right"),
    indicator=True,
    validate="many_to_one",  # The standard vlookup style merge
    update=False,  # This is a stata option for when there are merge conficts
    *args,
    **kwargs
) -> None:
    """
    From the stata manual:
    Because m:m merges are such a bad idea, we are not going to show you an example. If you think
    that you need an m:m merge, then you probably need to work with your data so that you can use a
    1:m or m:1 merge.

    By default, merge treats values from the master as inviolable. When observations match, it is the
    master’s values of the overlapping variables that are recorded in the merged result.

    If you specify the update option, however, then all missing values of overlapping variables in
    matched observations are replaced with values from the using data. Because of this new behavior,
    the merge codes change somewhat. Codes 1 and 2 keep their old meaning. Code 3 splits into codes
    3, 4, and 5. Codes 3, 4, and 5 are filtered according to the following rules; the first applicable rule
    is used.
    """
    # df = singleton.df
    # df = singleton.df
    labels = search_iterable(current.df.columns, varlist)
    # print(labels)

    def _merge(right: pd.DataFrame):
        return current.df.merge(
            right,
            on=labels,
            indicator=indicator,
            suffixes=suffixes,
            how=how,  # type: ignore
            validate=validate,  # The standard vlookup style merge
            *args,
            **kwargs
        )

    if right == "preserved":
        # right = current.df_preserved
        current.df = _merge(current.df_preserved)
        current.metadata["variable_labels"].update(
            current.metadata_preserved["variable_labels"]
        )
    elif not isinstance(right, str):  # DataFrame or named Series
        # current.df = current.df = _merge(right)
        current.df = _merge(right)
        # current.metadata["variable_labels"].update(
        #     current.metadata_preserved["variable_labels"]
        # )
    else:
        # current.df = current.df = _merge(_use(right)[0])
        right_df, right_metadata = _use(right)
        current.df = _merge(right_df)
        current.metadata["variable_labels"].update(right_metadata["variable_labels"])

    if update:
        pass  # under construction; see stata options

    _print(current.df)

pdexplorer/merge.py
pandichef/pdexplorer
from .._search import search_iterable
from .._dataset import current
from .._get_custom_attributes import _get_custom_attributes
from .._patsify import _patsify
from .._print import _print
from .._commandarg import parse

# from statsmodels.regression.linear_model import RegressionResultsWrapper
from typing import Union
import pandas as pd


def regressml(commandarg: str):
    """
    Stata docs: https://www.stata.com/manuals/rregress.pdf

    
    """
    from sklearn.linear_model import LinearRegression
    from sklearn.metrics import mean_squared_error, r2_score

    _ = parse(commandarg, "varlist")

    varlist_as_list = _["varlist"].split()
    df_dropna = current.df[varlist_as_list].dropna()

    yvar = varlist_as_list[0]
    xvars = varlist_as_list[1:]
    # xvars = search_iterable(current.df.columns, " ".join(xvars))
    X = df_dropna[xvars].values
    y = df_dropna[yvar].values
    # regr = LinearRegression()
    # regr.fit(X, y)
    results = LinearRegression().fit(X, y)

    # print(regr.coef_)
    # print(reg.coef_)
    LinearRegression.__call__ = lambda self: self.__dict__  # type: ignore
    predicted = results.predict(X)

    # results and predict command
    current.stored_results["e"] = {
        "scalars": {
            # "N": int(results.nobs),
            "df_m": results()["n_features_in_"],  # type: ignore
            # "df_r": int(results.df_resid),
            # "F": results.fvalue,
            "r2": r2_score(y, predicted),
            # "mss": mean_squared_error(predicted, y),
            # "r2_a": results.rsquared_adj,
            # "ll": results.llf,
        }
    }

    def predict_fnc(newvar: str) -> None:
        X = current.df[_["varlist"].split()[1:]]
        predictions_narray = results.predict(X)
        current.df[newvar] = pd.Series(predictions_narray)

    current.predict_fnc = predict_fnc

    return results

pdexplorer/ml/regressml.py
pandichef/pdexplorer
# https://journals.sagepub.com/doi/epdf/10.1177/1536867X1601600407
# https://scikit-learn.org/stable/modules/svm.html

pdexplorer/ml/svmachines.py
pandichef/pdexplorer
from .._commandarg import parse
from .._dataset import current
from .._search import search_iterable
from .._print import _print

graphviz_html = """
<!DOCTYPE html>
<html>
<head>
    <title>Graphviz Graph</title>
</head>
<body>
    <h1>My Graph</h1>
    <img src="{}" alt="Graph" />
</body>
</html>
"""


def treeclassify(
    commandarg: str, max_depth=None, show_graphviz=False, use_regressor=False
):
    """
    Ref: https://scikit-learn.org/stable/modules/tree.html
    
    It's challenging to get graphviz to work on Windows.  The comment here from Jyotsna_b helped:
    https://stackoverflow.com/questions/35064304/runtimeerror-make-sure-the-graphviz-executables-are-on-your-systems-path-aft
  
    """
    from sklearn import tree

    _ = parse(commandarg, "varlist")
    varlist_as_list = _["varlist"].split()
    yvar = varlist_as_list[0]
    xvars = varlist_as_list[1:]
    X = current.df.dropna()[xvars].values
    y = current.df.dropna()[yvar].values
    _DecisionTree = (
        tree.DecisionTreeRegressor if use_regressor else tree.DecisionTreeClassifier
    )
    decision_tree = _DecisionTree(random_state=0, max_depth=max_depth).fit(X, y)

    if show_graphviz:
        import graphviz
        from .._webbrowser import webbrowser_open

        dot_data = tree.export_graphviz(
            decision_tree,
            out_file=None,
            feature_names=xvars,
            class_names=list(current.df[yvar].cat.categories),
            filled=True,
            rounded=True,
            special_characters=True,
        )
        graph = graphviz.Source(dot_data)  # type: ignore
        graph.render("graphviz", format="svg")
        print("SVG file saved as 'graphviz.svg'")
        graph_viz_filename = "graphviz.html"
        with open(graph_viz_filename, "w") as f:
            f.write(graphviz_html.format("graphviz.svg"))
        webbrowser_open(graph_viz_filename)
    else:
        _print(tree.export_text(decision_tree, feature_names=xvars))

    return decision_tree

pdexplorer/ml/treeclassify.py
pandichef/pdexplorer
import warnings
from .._dataset import current
from .._print import _print


def gpt2(newvarname: str):
    # newvarname = namelist.split()[0]
    # textvar = namelist.split()[1]

    from transformers import pipeline

    generator = pipeline(model="gpt2", device=0)

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        current._df["gpt2"] = current._df[newvarname].apply(
            lambda x: generator(x)[0]["generated_text"]  # type: ignore
        )
    _print(current.df)

pdexplorer/nn/gpt2.py
pandichef/pdexplorer
from .._dataset import current
from .._print import _print
from .._commandarg import parse


def pipeline(
    # namelist: str,
    # task: str = "text-classification",
    # model_name: str = "distilbert-base-uncased",
    commandarg: str,
    output_dir="output_dir",
    # num_examples=100,
    tokenizer_model_name="distilbert-base-uncased",
    # num_labels=5,
    # run_in_sample=False,
):
    """Run pipeline on every row; deprecated"""
    _ = parse(commandarg, "varlist")
    assert len(_["varlist"].split()) == 1
    # yvar = _["varlist"].split()[0]
    xvar = _["varlist"].split()[0]

    from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline

    fine_tuned_text_classification_pipeline = pipeline(
        "text-classification",
        model=AutoModelForSequenceClassification.from_pretrained(output_dir),
        tokenizer=AutoTokenizer.from_pretrained(tokenizer_model_name),
    )

    def _fine_tuned_text_classification_pipeline(x):
        try:
            return int(
                fine_tuned_text_classification_pipeline(x)[0]["label"].replace(
                    "LABEL_", ""
                )
            )
        except RuntimeError:
            return -1

    current._df["_finetuned"] = current._df[xvar].apply(
        _fine_tuned_text_classification_pipeline
    )
    _print(current.df)
    # from transformers import pipeline

    # pipe_fnc = pipeline(task=task, model=model_name)  # fill-mask is default task #

    # assert len(namelist.split()) == 2
    # newvarname = namelist.split()[0]
    # inputvarname = namelist.split()[1]

    # def _pipeline(x):
    #     return pipe_fnc(x[:1000])[0]["label"]  # Error if >1940 or so #

    # current._df[newvarname] = current._df[inputvarname].apply(_pipeline)
    # # current._df["len"] = current._df[inputvarname].apply(len)
    # _print(current.df)

pdexplorer/nn/pipeline.py
pandichef/pdexplorer
# JUST A DEMO OF HOW LINEAR REGRESSION WORKS WITH NN #
from .._search import search_iterable
from .._dataset import current
from .._get_custom_attributes import _get_custom_attributes
from .._patsify import _patsify
from .._print import _print


# from statsmodels.regression.linear_model import RegressionResultsWrapper
from typing import Union


# def regressnn(varlist: str, library="pytorch", epochs: int = 100):
def regressnn(varlist: str, use_dataloader=False, epochs: int = 100):
    # ) -> Union[RegressionResultsWrapper, LinearRegression, None]:
    """
    Stata docs: https://www.stata.com/manuals/rregress.pdf
    Returns: https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.RegressionResults.html
    """
    # if library == "statsmodels":
    #     import statsmodels.formula.api as smf

    #     df = current.df
    #     patsy_formula = _patsify(varlist)
    #     # print(patsy_formula)
    #     model = smf.ols(patsy_formula, data=df, missing="drop")
    #     results = model.fit()
    #     _print(results.summary())
    #     current.methods, current.properties = _get_custom_attributes(results)
    #     # print(results)
    #     # print(current.properties)
    #     current.stored_results["e"] = {}
    #     current.stored_results["e"] = {
    #         "scalars": {
    #             "N": int(results.nobs),
    #             "df_m": int(results.df_model),
    #             "df_r": int(results.df_resid),
    #             "F": results.fvalue,
    #             "r2": results.rsquared,
    #             "mss": results.mse_model,
    #             "r2_a": results.rsquared_adj,
    #             "ll": results.llf,
    #         }
    #     }
    #     return results
    # elif library == "sklearn" or library == "scikit-learn":
    #     from sklearn.linear_model import LinearRegression

    #     # import numpy as np

    #     varlist_as_list = varlist.split()
    #     yvar = varlist_as_list[0]
    #     xvars = varlist_as_list[1:]
    #     xvars = search_iterable(current.df.columns, " ".join(xvars))
    #     X = current.df.dropna()[xvars].values
    #     y = current.df.dropna()[yvar].values
    #     reg = LinearRegression().fit(X, y)
    #     return reg
    if not use_dataloader:
        # elif library == "pytorch":
        # Source: https://towardsdatascience.com/linear-regression-with-pytorch-eb6dedead817
        # This didn't work with SDG, so I used Rprop instead
        import numpy as np
        import torch
        from torch.autograd import Variable

        # class linearRegression(torch.nn.Module):
        #     def __init__(self, inputSize, outputSize):
        #         super(linearRegression, self).__init__()
        #         self.linear = torch.nn.Linear(inputSize, outputSize)

        #     def forward(self, x):
        #         out = self.linear(x)
        #         return out

        # if varlist == "":
        #     # create dummy data for training
        #     x_values = [i for i in range(11)]
        #     x_train = np.array(x_values, dtype=np.float32)
        #     x_train = x_train.reshape(-1, 1)

        #     y_values = [2 * i + 1 for i in x_values]
        #     y_train = np.array(y_values, dtype=np.float32)
        #     y_train = y_train.reshape(-1, 1)
        #     X = torch.tensor(x_train)
        #     y = torch.tensor(y_train)
        # else:
        varlist_as_list = varlist.split()
        yvar = varlist_as_list[0]
        xvars = varlist_as_list[1:]
        xvars = search_iterable(current.df.columns, " ".join(xvars))
        x_train = current.df.dropna()[xvars].values
        y_train = current.df.dropna()[[yvar]].values
        X = torch.tensor(x_train).to(torch.float32)
        y = torch.tensor(y_train).to(torch.float32)

        # print(X.dtype)
        # print(y.dtype)
        # print(X.shape)
        # print(y.shape)
        # print(X.shape[1])
        # print(X)
        # print(y)

        inputDim = X.shape[1]  # takes variable 'x'
        outputDim = 1  # takes variable 'y'
        learningRate = 0.01
        epochs = epochs

        # model = linearRegression(inputDim, outputDim)
        model = torch.nn.Linear(inputDim, outputDim)
        ##### For GPU #######
        if torch.cuda.is_available():
            model.cuda()
        criterion = torch.nn.MSELoss()
        optimizer = torch.optim.Rprop(model.parameters(), lr=learningRate)

        for epoch in range(epochs):
            # Converting inputs and labels to Variable
            if torch.cuda.is_available():
                inputs = X.cuda()
                labels = y.cuda()
            else:
                inputs = X
                labels = y

            # print(inputs)
            # print(labels)

            # Clear gradient buffers because we don't want any gradient from previous epoch to carry forward, dont want to cummulate gradients
            optimizer.zero_grad()

            # get output from the model, given the inputs
            outputs = model(inputs)

            # get loss for the predicted output
            loss = criterion(outputs, labels)
            # print(loss)
            # get gradients w.r.t to parameters
            loss.backward()

            # update parameters
            optimizer.step()

            # print("epoch {}, loss {}".format(epoch, loss.item()))

        """
        with torch.no_grad():  # we don't need gradients in the testing phase
            if torch.cuda.is_available():
                predicted = model(X.cuda()).cpu().data.numpy()
            else:
                predicted = model(X).data.numpy()
            # print(predicted)

        plt.clf()
        plt.plot(x_train, y_train, "go", label="True data", alpha=0.5)
        plt.plot(x_train, predicted, "--", label="Predictions", alpha=0.5)
        plt.legend(loc="best")
        plt.show()
        """

        return model  # type: ignore
    else:
        # elif library == "pytorch_dataloader":
        # same as above, but using the dataloader
        import torch
        from torch.autograd import Variable

        ds = current.get_pytorch_dataset(varlist)
        dl = current.get_pytorch_dataloader(varlist)

        # print(ds.x_train.shape)
        # print(ds.y_train.shape)
        # print(y.shape)

        inputDim = ds.x_train.shape[1]  # takes variable 'x'
        outputDim = 1  # takes variable 'y'
        learningRate = 0.01
        epochs = epochs

        model = torch.nn.Linear(inputDim, outputDim)
        ##### For GPU #######
        if torch.cuda.is_available():
            model.cuda()
        criterion = torch.nn.MSELoss()
        optimizer = torch.optim.Rprop(model.parameters(), lr=learningRate)

        # for X, y in dl:
        #     print(X)
        #     print(y)
        #     break

        for epoch in range(epochs):
            for X, y in dl:
                # print(X.shape)
                # Converting inputs and labels to Variable
                if torch.cuda.is_available():
                    inputs = X.cuda()
                    labels = y.cuda()
                else:
                    inputs = X
                    labels = y

                # print(inputs)
                # print(labels)

                # Clear gradient buffers because we don't want any gradient from previous epoch to carry forward, dont want to cummulate gradients
                optimizer.zero_grad()

                # get output from the model, given the inputs
                outputs = model(inputs)

                # get loss for the predicted output
                loss = criterion(outputs, labels)
                # print(loss)
                # get gradients w.r.t to parameters
                loss.backward()

                # update parameters
                optimizer.step()

                # print("epoch {}, loss {}".format(epoch, loss.item()))

        # for epoch in range(epochs):
        #     for batch, (X, y) in enumerate(dl):
        #         # Converting inputs and labels to Variable
        #         if torch.cuda.is_available():
        #             inputs = X.cuda()
        #             labels = y.cuda()
        #         else:
        #             inputs = X
        #             labels = y
        #         optimizer.zero_grad()
        #         outputs = model(inputs)
        #         loss = criterion(outputs, labels)
        #         loss.backward()
        #         optimizer.step()

        return model  # type: ignore
    # else:
    # raise Exception("Invalid library name.")


"""
def regress2(varlist: str):
    # https://www.stata.com/manuals/rregress.pdf
    # page 6
    # regress mpg weight foreign
    df = singleton.df

    varlist_as_list = varlist.split()
    yvar = varlist_as_list[0]
    xvars = varlist_as_list[1:]

    xvars = search_iterable(df.columns, " ".join(xvars))

    for col in df.columns:
        if isinstance(df[col].dtype, CategoricalDtype):
            df[col] = df[col].cat.codes

    X = df[xvars]
    y = df[yvar]

    X = sm.add_constant(X)
    results = sm.OLS(y, X).fit()
    print(results.summary())
    # print(type(results))
    # print(results.params)
    # return get_custom_attributes(results)
    singleton.methods, singleton.properties = get_custom_attributes(results)
    # pprint(custom_methods)
    # pprint(custom_properties)
    return results
"""


pdexplorer/nn/regressnn.py
pandichef/pdexplorer
from ._search import search_iterable
from ._dataset import current
from ._print import _print


def order(varlist, refvar=None, after=True, last=False) -> None:
    """
    For whatever reason, it isn't that simple to order columns in pandas.
    This method corrects that.  Note: This currently only supports column
    ordering.

    Parameters
    ----------
    varlist : single label or list-like or magicpandas search string
        The columns to order.
    refvar str, default None
        A reference column e.g., to order after or before refvar
    after : bool, default True
        If True, order after the refvar
    last : bool, default False
        If True, place ordered columns at the end of the dataframe

    Returns
    -------
    MagicDataFrame
        DataFrame with ordered columns.
    """
    df = current.df
    # if type(varlist) == str:
    #     varlist = varlist.split()
    # varlist = df._parse_varlist(varlist)
    varlist = search_iterable(df.columns, varlist)

    assert type(after) == bool
    assert type(last) == bool
    assert (
        last is False or refvar is None
    ), "last and refvar options cannot be use simultaneously"

    therest = [x for x in list(df.columns) if x not in varlist]

    if refvar is None:
        if len(therest) != 0:
            refvar = therest[0]
        after = False

    if last is True:
        after = True
        refvar = therest[-1]

    if len(therest) != 0:
        colnum = list(therest).index(refvar)  # type: ignore
    else:
        colnum = 0

    if after is False:
        head = therest[:colnum]
        tail = therest[colnum:]
    else:
        head = therest[: colnum + 1]
        tail = therest[colnum + 1 :]

    newlist = head
    newlist.extend(varlist)
    newlist.extend(tail)
    df = df[newlist]
    # return df
    current.df = df
    _print(current.df)

pdexplorer/order.py
pandichef/pdexplorer
# UNDER CONSTRUCTION
# It currently works, but doesn't support the Stata API
# See https://www.stata.com/products/stb/journals/stb24.pdf
# cf https://pandas.pydata.org/docs/reference/api/pandas.cut.html
# orig implementation: https://github.com/pandichef/magicpandas/blob/master/magicpandas/frame.py#L148
import re
import numpy as np
import pandas as pd
from typing import Optional
from .._dataset import current
from .._print import _print
from pandas.api.types import is_integer_dtype, is_float_dtype

# def pcut(self, step_size: float = None, bin_count: int = None,
#             right: bool = True, format: str = '{:3.2f}') -> MagicSeries:
def srecode(
    newvar,
    varname,
    step_size: Optional[float] = None,
    bin_count: Optional[int] = None,
    right: bool = True,
    format: str = "{:3.2f}",
) -> None:
    """Converts pd.Series of dtype float or int into a ordinal pd.Series of
    dtype category that is formatted in a manner suitable for publishing in
    non-scientific contexts e.g., "1.001 - 2.000" vs (1.001, 2.000].

    TODO: EDIT API SO IT IS CONSISTENT WITH THE STATA COMMAND

    Parameters
    ----------
    step_size : float
        The desired number of steps.  Note: must be None if bin_count is
        not None.
    bin_count : int
        The desired number of bins.  Note: must be None if step_size is
        not None.
    right : bool
        Indicates whether bins includes the rightmost edge or not.
    format : str
        Python style format string e.g., {:3.2f}

    Returns
    -------
    None
    """
    self = current.df[varname]  # self is a Series

    # print(self)

    if not is_float_dtype(self) and not is_integer_dtype(self):
        raise TypeError("Series must of dtype float or int")

    if step_size is not None and bin_count is not None:
        raise ValueError("step_size is not None and bin_count is not None")
    elif step_size is None and bin_count is None:
        raise ValueError("step_size is None and bin_count is None")

    if bin_count is not None:
        step_size = (self.max() - self.min()) / (bin_count - 1)
        step_size = round(step_size, 10)  # type: ignore

    thisformat = (
        format.replace("{:", "{0:") + " - " + format.replace("{:", "{1:")
    )  # e.g., "{0:3.1f} - {1:3.1f}"
    delta = 1 / 10 ** int(
        re.findall(r"\.\d+", format)[0].replace(".", "")
    )  # e.g., 0.01

    minvalue = int(self.min() / step_size) * step_size  # type: ignore
    maxvalue = int(self.max() / step_size + 1) * step_size  # type: ignore

    if right:
        bin_labels = [
            thisformat.format(i + delta, i + step_size)
            for i in np.arange(minvalue, maxvalue, step_size)
        ]
        bin_labels[0] = thisformat.format(minvalue, minvalue + step_size)  # type: ignore
    else:
        bin_labels = [
            thisformat.format(i, i + step_size - delta)
            for i in np.arange(minvalue, maxvalue, step_size)
        ]

    bins = np.arange(minvalue, maxvalue + step_size, step_size)  # type: ignore

    # Below is a hack to handle "ValueError: Bin labels must be one fewer
    # than the number of bin edges."
    if len(bin_labels) == len(bins):
        bin_labels = bin_labels[:-1]

    current._df[newvar] = pd.cut(
        self, bins=bins, right=right, labels=bin_labels, include_lowest=True
    )

    # print(self)
    _print(current.df)

pdexplorer/plus/srecode.py
pandichef/pdexplorer
from ._dataset import current
from ._print import _print


def predict(newvar: str) -> None:
    if current.predict_fnc:
        current.predict_fnc(newvar)
        current.metadata["variable_labels"][newvar] = "Fitted values"
    else:
        _print("last estimates not found")

pdexplorer/predict.py
pandichef/pdexplorer
# from ._singleton import singleton
from ._dataset import current


def preserve(quietly=True):
    current.df_preserved = current.df
    current.metadata_preserved = current.metadata
    current.has_preserved = True

    if not quietly:
        print(current.df)

pdexplorer/preserve.py
pandichef/pdexplorer
# raw_syntax = """[anything] [if] [in] [aw fw iw pw] [,		///
raw_syntax = """varlist [if] [in] [aw fw iw pw] [,		///
	VCE(passthru) Robust CLuster(passthru)		///
	HC2 HC3						///
	beta						///
	EForm(passthru)					///
	noHEader					///
	noTABle						///
	plus						///
	dfci DFPValues					///
	*						///
]"""
# https://www.stata.com/manuals13/psyntax.pdf

import pandas as pd
from ._search import search_iterable
from ._dataset import current
from ._get_custom_attributes import _get_custom_attributes
from ._patsify import _patsify
from ._print import _print
from ._commandarg import parse

# from statsmodels.regression.linear_model import RegressionResultsWrapper
from typing import Union


def regress(commandarg: str):
    """
    Stata docs: https://www.stata.com/manuals/rregress.pdf
    Returns: https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.RegressionResults.html
    """
    # assert False, current.df
    syntax = " ".join(raw_syntax.replace("///", "").split())
    _ = parse(commandarg, syntax)
    varlist = _["varlist"]

    import statsmodels.formula.api as smf
    import statsmodels.api as sm

    # df = current.df
    patsy_formula = _patsify(varlist)
    if _["if"]:
        model = smf.ols(patsy_formula, data=current.df.query(_["if"]), missing="drop")
    else:
        model = smf.ols(patsy_formula, data=current.df, missing="drop")
    results = model.fit()
    _print(results.summary2())
    current.methods, current.properties = _get_custom_attributes(results)  # deprecated

    # results and predict command
    # current.stored_results["e"] = {}
    current.stored_results["e"] = {
        "scalars": {
            "N": int(results.nobs),
            "df_m": int(results.df_model),
            "df_r": int(results.df_resid),
            "F": results.fvalue,
            "r2": results.rsquared,
            "mss": results.mse_model,
            "r2_a": results.rsquared_adj,
            "ll": results.llf,
        }
    }

    def predict_fnc(newvar: str) -> None:
        X = current.df[_["varlist"].split()[1:]]
        predictions_narray = results.predict(sm.add_constant(X))
        current.df[newvar] = pd.Series(predictions_narray)

    current.predict_fnc = predict_fnc

    # def to_dict(self):
    #     attr_dict = {
    #         attr_name: getattr(self, attr_name)
    #         for attr_name in dir(self)
    #         if not callable(getattr(self, attr_name))
    #     }
    #     return attr_dict

    results.__class__.__call__ = lambda self: self.__dict__["_results"].__dict__  # type: ignore
    return results

pdexplorer/regress.py
pandichef/pdexplorer
from ._dataset import current
from ._print import _print


def rename(oldnew: str) -> None:
    def _change_chase(string_method):
        column_mapping = {}
        for col in current.df.columns:
            if isinstance(col, str):  # type error without this
                column_mapping[col] = string_method(col)
            else:
                column_mapping[col] = col
        current.df.rename(columns=column_mapping, inplace=True)

    # todo: handle any set of variable names
    if oldnew.replace(" ", "") == "*,lower":
        _change_chase(str.lower)
    elif oldnew.replace(" ", "") == "*,upper":
        _change_chase(str.upper)
    elif oldnew.replace(" ", "") == "*,proper":
        _change_chase(str.title)
    else:
        df = current.df
        oldnew_split = oldnew.strip().split(" ")
        assert (
            len(oldnew_split) == 2
        ), "Function parameter must be the old variable name followed by the new variable name."
        df.rename(columns={oldnew_split[0]: oldnew_split[1]}, inplace=True)
    _print(current.df)

pdexplorer/rename.py
pandichef/pdexplorer
# TODO: add in_/if_
from ._dataset import current
from ._print import _print
from ._commandarg import parse


def replace(commandarg: str) -> None:

    _ = parse(commandarg)

    assignment = _["anything"] + "=" + _["="]

    new_column_name = assignment.split("=")[0].strip()
    assert (
        new_column_name in current.df.columns
    ), 'Column already exists.  See "generate" command.'
    if _["if"]:
        current.df.loc[current.df.query(_["if"]).index, _["anything"],] = _["="]
    else:
        current.df.eval(assignment, inplace=True)
    _print(current.df)

pdexplorer/replace.py
pandichef/pdexplorer
# under construction
import numpy as np
from ._search import search_iterable
from ._dataset import current
from ._print import _print
import pandas as pd

"""
pandas has several reshaping functions, including df.unstack('level') for going to wide, 
df.stack('column_level') for going to long, pd.melt, and df.pivot.

See https://pandas.pydata.org/pandas-docs/stable/user_guide/reshaping.html
See https://www.stata.com/manuals/dreshape.pdf

pandas "record format" = stata "wide format"
pandas "stacked format" = stata "long format"
"""


def reshapewide(stub, i, j):
    current.df[j] = stub + current.df[j].astype(str)
    current.df = current.df.pivot(index=i, columns=j, values=stub)
    current.df = current.df.reset_index()
    current.df.index.name = None
    current.df.columns.name = None
    _print(current.df)


def reshapelong(stub, i, j):
    # varlist = search_iterable(singleton.df.columns, stub + "*")
    # singleton.df.melt(id_vars=varlist, var_name=j, value_name=stub)
    current.df = pd.wide_to_long(current.df, [stub], i=i, j=j)
    current.df.sort_values(by=[i, j], inplace=True)
    current.df.reset_index(inplace=True)
    _print(current.df)

pdexplorer/reshape.py
pandichef/pdexplorer
import pandas as pd
from ._dataset import current
from ._print import _print


def restore(swap=True):
    if swap:
        current.df, current.df_preserved = current.df_preserved, current.df
        current.metadata, current.metadata_preserved = (
            current.metadata_preserved,
            current.metadata,
        )
    else:
        current.df = current.df_preserved
        current.metadata = current.metadata_preserved
        current.df_preserved = pd.DataFrame()
        current.metadata_preserved = {}
        current.has_preserved = False

    _print(current.df)

pdexplorer/restore.py
pandichef/pdexplorer
from typing import Literal
from ._dataset import current
from pprint import pprint

###############################################
# r-class
def returnlist(return_type: Literal["r", "e", "s", "n", "c"] = "r") -> None:
    # dct = current.stored_results["e"]
    print()
    for values_type, values_dict in current.stored_results[return_type].items():
        print(values_type + ":")
        for k, v in values_dict.items():
            v_as_str = str(k)
            shift = " " * (16 - len(v_as_str))
            print(shift + "_e('" + k + "') =  " + str(v))


def r_(name, return_type: Literal["r", "e", "s", "n", "c"] = "r"):
    flattened_dict = {}
    for key, value in current.stored_results["e"].items():
        flattened_dict.update(value)
    return flattened_dict[name]


###############################################
# e-class
def ereturnlist() -> None:
    returnlist("e")


def e_(name):
    return r_(name, "e")


###############################################
# s-class
def sreturnlist() -> None:
    returnlist("s")


def s_(name):
    return r_(name, "s")


###############################################
# n-class
def nreturnlist() -> None:
    returnlist("n")


def n_(name):
    return r_(name, "n")


###############################################
# c-class
def creturnlist() -> None:
    returnlist("c")


def c_(name):
    return r_(name, "c")

pdexplorer/returnlist.py
pandichef/pdexplorer
from ._dataset import current
from ._print import _print


def sample(percentage_or_count, by=None, count=False, random_state=None):
    if count:
        kwargs = {"n": percentage_or_count}
    else:
        kwargs = {"frac": percentage_or_count / 100}
    if random_state is not None:
        kwargs.update({"random_state": random_state})

    if by:
        current.df = (
            current.df.groupby(by)
            .apply(lambda x: x.sample(**kwargs))
            .reset_index(drop=True)
        )
    else:
        current.df = current.df.sample(**kwargs)
    _print(current.df)

pdexplorer/sample.py
pandichef/pdexplorer
import os
from ._dataset import current
import pandas as pd
from .lis import lis
from ._print import _print


def save(file_path=None, use_variable_labels=False):
    if use_variable_labels:
        column_labels = current.metadata["variable_labels"]
    else:
        column_labels = {}

    # df = singleton.df
    if not file_path:
        current.df.rename(columns=column_labels).to_clipboard()
        _print("(Data saved to clipboard)")
    elif os.path.splitext(file_path)[1] == "":
        current.get_huggingface_dataset().save_to_disk(file_path)
    else:
        assert (
            not use_variable_labels
        ), "dta files save variable labels as metadata.  Set use_variable_labels to False."
        file_extension = os.path.splitext(file_path)[1]
        if file_extension == ".dta":
            current_df_copy = current.df.copy()
            for catcol in current_df_copy.select_dtypes(
                include=["category"]
            ).columns.tolist():
                current_df_copy[catcol] = current_df_copy[catcol].cat.codes
            current_df_copy.to_stata(
                file_path,
                data_label=current.metadata["data_label"],
                variable_labels=current.metadata["variable_labels"],
                value_labels=current.metadata["value_labels"],
                write_index=False,
                version=118,
            )
        elif file_extension == ".csv":
            current.df.to_csv(file_path)
        elif file_extension in [".xlsx", ".xls"]:
            current.df.to_excel(file_path)
        elif file_extension in [".pkl"]:
            import pickle

            with open(file_path, "wb") as file:
                # Serialize and save the object to the file
                pickle.dump(current, file)
        else:
            print("Didn't save file.  Something went wrong.")

pdexplorer/save.py
pandichef/pdexplorer
# Inspiration: https://xlsxwriter.readthedocs.io/working_with_pandas.html
# Note the complexity in handling the header in pandas
import pandas as pd
import subprocess
import os
from .._dataset import current
from typing import Optional


def dfedit(use_labels=False):
    import dtale  # very slow import

    if current.dtale_browser:
        current.dtale_browser.kill()
    if use_labels:
        # Note: for whatever reason, the "name" attribute cannot contain underscores
        new_dtale_browser = dtale.show(current.df_labeled, name="currentdflabeled")
    else:
        new_dtale_browser = dtale.show(current.df, name="currentdf")
    # if current.dtale_browser is None:
    new_dtale_browser.open_browser()
    current.dtale_browser = new_dtale_browser
    # current.dtale_browser = dtale.show(current.df_labeled, name="currentdf")
    # current.dtale_browser.open_browser()


# def browse_in_excel(
#     path: Optional[str] = None,
#     limit: Optional[int] = None,
#     run: bool = True,
#     percentage_columns: Optional[list] = None,
#     autofit_columns: Optional[list] = None,
# ) -> None:
#     df = current.df

#     if not path:
#         # set default file i.e., tmp/easy_xlsxwriter.xlsx
#         import tempfile

#         path = os.path.join(tempfile.gettempdir(), "easy_xlsxwriter.xlsx")

#     writer = pd.ExcelWriter(path=path, engine="xlsxwriter")  # type: ignore
#     if limit:
#         df.head(limit).to_excel(writer, sheet_name="Sheet1", startrow=1, header=False)
#     else:
#         df.to_excel(writer, sheet_name="Sheet1", startrow=1, header=False)
#     workbook = writer.book
#     worksheet = writer.sheets["Sheet1"]

#     default_format = workbook.add_format(  # type: ignore
#         {
#             "num_format": "#,##0.00",
#             "align": "center",
#             # 'shrink': True,
#         }
#     )
#     for column in df.columns:
#         this_column_position = list(df.columns).index(column) + 1
#         worksheet.set_column(
#             this_column_position, this_column_position, cell_format=default_format
#         )

#     percentage_format = workbook.add_format({"num_format": "0%", "align": "center"})  # type: ignore
#     if percentage_columns:
#         for column in percentage_columns:
#             this_column_position = list(df.columns).index(column) + 1
#             worksheet.set_column(
#                 this_column_position,
#                 this_column_position,
#                 cell_format=percentage_format,
#             )

#     # format header
#     header_format = workbook.add_format(  # type: ignore
#         {
#             "bold": True,
#             "text_wrap": True,
#             "align": "center",
#             "valign": "top",
#             "fg_color": "#D7E4BC",
#             "border": 1,
#         }
#     )
#     for col_num, value in enumerate(df.columns.values):
#         worksheet.write(0, col_num + 1, value, header_format)
#     worksheet.write(0, 0, "Ticker", header_format)
#     worksheet.freeze_panes(1, 1)

#     # autofit
#     if autofit_columns:
#         for column in autofit_columns:
#             column_width = df[column].str.len().max()
#             this_column_position = list(df.columns).index(column) + 1
#             worksheet.set_column(
#                 this_column_position, this_column_position, width=column_width
#             )

#     # save excel file
#     worksheet.set_zoom(75)
#     writer.close()  # type: ignore
#     if run:
#         subprocess.run(f"start excel {path}", shell=True)

pdexplorer/shortcuts/dfedit.py
pandichef/pdexplorer
import altair as alt
from .._altair_mapper import barchart  # type: ignore


def histogram(varname) -> alt.Chart:
    return barchart().encode(alt.X(varname, bin=True), y="count()")

pdexplorer/shortcuts/histogram.py
pandichef/pdexplorer
import pandas as pd
from .._commandarg import parse, parse_options
from .._dataset import current
from .._print import _print


def melt(commandarg, use_labels=True):
    # e.g., melt("miles_per_gallon horsepower,  keep(weight_in_lbs)")
    _ = parse(commandarg)
    parsed_options = parse_options(_["options"])
    value_vars = _["anything"].split()
    id_vars = parsed_options["keep"].split()
    if use_labels:
        for var in value_vars:
            current._df = current._df.rename(
                columns={var: current.metadata["variable_labels"][var]}
            )
        value_vars = list(
            map(lambda x: current.metadata["variable_labels"][x], value_vars)
        )

    current._df = pd.melt(current._df, id_vars=id_vars, value_vars=value_vars)
    _print(current._df)

pdexplorer/shortcuts/melt.py
pandichef/pdexplorer
def profile(varlist=None, package="ydata_profiler"):
    from .._profile import ydata_profile, sweetviz_profile

    if package == "ydata_profiler":
        ydata_profile(varlist=varlist)
    elif package == "sweetviz":
        sweetviz_profile(varlist=varlist)
    else:
        raise Exception(f"Package {package} not found.")

pdexplorer/shortcuts/profile.py
pandichef/pdexplorer
from .._altair_mapper import circlechart  # type: ignore

scatter = circlechart

# def scatter(anything) -> None:
#     raise NotImplementedError(
#         "Stata's scatter command isn't implemeted.  Try circlechart or pointchart instead"
#     )

pdexplorer/shortcuts/scatter.py
pandichef/pdexplorer
from ._dataset import current
from ._search import search_iterable
from ._print import _print


def sort(varlist, ascending=True):
    """See also gsort"""
    sort_list = search_iterable(current.df.columns, varlist)
    current.df.sort_values(sort_list, ascending=ascending, inplace=True)
    _print(current.df)

pdexplorer/sort.py
pandichef/pdexplorer
from ._search import search_iterable
from ._dataset import current
from ._by import byable
from ._print import _print


@byable
def summarize(varlist=None) -> None:
    # _supports_byvar()
    # if current.within_by_context_manager:
    #     raise WithinByContextManager

    df = current.df
    if varlist:
        labels = search_iterable(df.columns, varlist)
        inverted_labels = [x for x in df.columns if x not in labels]
        df = df.drop(labels=inverted_labels, axis=1, inplace=False)
    _print(df.describe(include="all").transpose())

pdexplorer/summarize.py
pandichef/pdexplorer
from .webuse import webuse


def sysuse(*args, **kwargs):
    webuse(*args, **kwargs, use_local=True)

pdexplorer/sysuse.py
pandichef/pdexplorer
import warnings
from ._search import search_iterable
from ._dataset import current
import pandas as pd
from ._print import _print


def tabulate(varlist):
    _varlist = varlist.split()
    if len(_varlist) == 1:
        varname = varlist
        df = current.df
        frequency_table = df.groupby(varname).size().reset_index(name="Freq")
        total_count = frequency_table["Freq"].sum()
        frequency_table["Percentage"] = (frequency_table["Freq"] / total_count) * 100
        # frequency_table = frequency_table.set_index(varname)
        frequency_table["Cum"] = frequency_table["Percentage"].cumsum()
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            pd.set_option("display.max_rows", None)
            _print(frequency_table)
            pd.reset_option("all")
        _print(f"*** Total Freq: {frequency_table['Freq'].sum()} ***")
    elif len(_varlist) == 2:
        twoway = pd.crosstab(current.df[_varlist[0]], current.df[_varlist[1]])  # type: ignore
        # twoway = pd.DataFrame(twoway.to_dict())  # hack to clean indexes
        twoway["Total"] = twoway.sum(axis=1).astype(int)
        _print(twoway)
    else:
        raise Exception("tabulate can only accept a varlist with two variables.")

pdexplorer/tabulate.py
pandichef/pdexplorer
import pandas as pd
import numpy as np

df1 = pd.DataFrame(
    {
        "rate": [5, 4.5, 4, 3.5, 3],
        "origfixedterm": [60, 84, 120, 180, 360],
        "origterm": [360, 360, 360, 180, 360],
        "origltv": [50, 60, 70, 80, 90],
        "origfico": [620, 640, 660, 680, 700],
        "age": [12, 12, 12, 12, 12],
        "origupb": [100, 200, 300, 400, 500],
        "upb": [100, 200, 300, 400, 500],
    }
)

yelp_reviews = [
    {
        "stars": 4,
        "text": "Food was very good, loved the guacamole dip, not a huge fan of the salsa for the chips.   Salsa is a bit peppery.    Really enjoyed the food though, this place just opened early August, 2011.   I will be back for sure, very friendly staff, not the greatest looking place but not bad.",
    },
    {
        "stars": 1,
        "text": "This place is tucked away in between a busy Starbucks and a large grocery store, among other small mom and pop shops. I needed a quick bite to eat and didn't want to deal with the line at Starbucks so I decided to pay a visit.\\n\\nThere's a huge poster in the entrance that advertises their $3 special for bagel with cream cheese and coffee. Bingo! I made my way in to divulge in their special. The dining area isn't anything extravagant, it's just what you need with lots of tables and chairs. Walking up to the order stand, I was quite disappointed to see that three-fourths of their bagel selection wasn't available. It was 10:50 AM so I wasn't sure that they just sold out of everything earlier in the morning or if they don't offer a large selection as listed. I opted for a simple blueberry bagel with cream cheese. \\n\\nUnlike Einstein Brothers where you can see the preparation of your food, this place has a huge wall from the order stand to the cashier's stand so preparation isn't visible. Whatever, I figure you can't mess up a bagel with cream cheese. They toasted my bagel (I didn't ask for this but I think I'd be annoyed if I didn't want it toasted) so that it had a nice crunch. But the bagel had absolutely no flavor. Blueberries? Nah, this bagel had me scratching my head wondering where the blueberry good ran off to.\\n\\nThey serve regular coffee as well as vanilla flavored coffee. Their vanilla coffee was pretty good, can't complain about that. \\n\\nAs the name states, this place serves gyros and deli sandwiches. Their \\\"New York\\\" bagels didn't impress me so I'm not sure if I'll be back anytime soon to try other items on their menu.  Sad face.",
    },
    {"stars": 4, "text": "Great place, great service and friendly people"},
    {
        "stars": 1,
        "text": "After the UYE @ Herbs and Rye was over there was talk of tacos, the crew all ended up going to another destination, but since I'm lazy I figured I'd check out the place 10 feet from my parked car.\\n\\nThe place is nothing more than a tiny shack with a big light up screen on top of it, they have menus on both sides of the windows, one in english and one in spanish. \\n\\nThey don't have a ton of variety, but everything is hand made, the tortillas are hand made, and mine were made to order.\\n\\nThe place wasn't really busy, which concerned me a little, but I thought I'd give it a try.\\n\\nI ordered a Carne Asada Taco and a Taco Al Pastor. I was a little surprised when I got my order, all there was on my plate was 2 tortillas with some meat on them, no lettuce, no cheese, nothing.\\n\\nThey do have a small toppings bar next to the window that had 2 salsas and a guac sauce, limes, cabbage, and jalapenos, on it, but I'm not really a fan of any of them.\\n\\nI squeezed some lime on to my meaty tortillas and gave them a try.\\n\\nThe Taco Al Pastor was pretty solid, the pork was cooked and seasoned well, but it was nothing special.\\n\\nThe Carne Asada Taco was rather disappointing, the beef was chewy and barely seasoned.\\n\\nThe tortillas were fresh and tasted pretty good, but I have to admit for $1.75 each I was expecting more than meat and a tortilla.\\n\\nThe only things saving this place from a 1 star review are the fact that the ladies were friendly, the food came out quick, and the al pastor was decent.\\n\\nIt will do in a pinch, but there are 3 other taco places within a block, so I'd say check them out first.\\n\\nOh and there are no tables or chairs or anywhere to sit, I had to sit in my car and eat my tacos",
    },
    {
        "stars": 0,
        "text": "I agree with Insider Pages, the staff is super rude for either dine-in or to-go, I guess I am not pretty enough or the right color. Beware of cashier 19. \\nThe manager is controlled by the staff. In this economy with those prices and that attitude: NO THANK YOU! Never again. \\nTry DaVinci Pizza down the road, the owner takes pride in his pizza and the taste is almost as good for half the price.",
    },
    {
        "stars": 0,
        "text": "I have NEVER written a review on ANYTHING, but I absolutely had to for this restaurant. I cooked Chinese food for years and preparing this food in such a bland way is hard to do, so bravo guys. I've ate here a few times and it's either ok or just plain not good, and I mean not good to the point where I will literally just toss it out rather than eat it. So take my review for what it is. I know the business and one review is not gonna make or break a restaurant, but they have definitely lost my family, friends and I as customers.",
    },
    {
        "stars": 0,
        "text": "I love cafe rio but this location was very disappointing. The staff was very rude and the salad that I ordered comes with cheese but the guy only gave me a pinch and told me if I wanted more than that it would cost $1 more. The other locations give you cheese and don't argue with their customers over it. Then I asked for limes and they gave me lemons instead and when I told then, he rolled his eyes at me, like really. This location is very stingy with their ingredients and the staff have poor attitudes. I Will NEVER be back to this location EVER again and I have advised several others to go to a different location so they don't go through the same hassle I did. Lack of customer service is prevalent here.\\nFYI: If you hate your job that much that you treat your customers like shit, then quit your job. Management shouldn't allow this to happen, PERIOD!!!!!!!",
    },
    {
        "stars": 1,
        "text": "My husband and I went to the Cheesecake Factory for lunch. PROS- no wait, friendly waiter, large menu. CONS- atmosphere was very warehouse/industrial, food was overpriced. We will not be going back.",
    },
    {
        "stars": 3,
        "text": "Bartender was knowledgeable, friendly and knew how to make my favorite drink (a boulevardier) without having to look it up.  Upon ordering my second one, I made a suggestion as to how he could improve it (more compari) and he did as told and the second drink was even better.\\n\\nI ordered the pig ears for curiosity's sake.  The texture was slightly off-putting (the portion size was too big), but they were well seasoned and the salsa that came with it was very delicious.",
    },
    {
        "stars": 2,
        "text": "This place certainly isn't new and fancy like many of the other locations on the strip, but it is an okay place to get a good night's rest. There were a lot of children running around and things were a bit dingy and run down, but the price is pretty awesome. Do beware of the additional resort fees, which do get you a coupon book with some decent stuff like a free premium ride at the adventure dome and 2 midway games, plus some deals on drinking and gambling. Resort fee also covered free wifi. There is a gym which was much nicer than I expected, but very difficult to find. Many people asked seem to not know where it was hidden and there are no helpful signs. I would suggest it be kept open 24 hours a day since this is the city that never sleeps, but the hours started early and ended semi late. The pool didn't appear to be anything worth checking out unless you are super hot, especially if you have used any of the other pools on the strip. We did have a problem with an alarm clock next door going off in the middle of the night, but staff did get it taken care of eventually. I also had a problem with my safe, which may have been forgetting my own code, and was impressed when security arrived prior to the maintenance man to verify my identification. Do beware the food at the buffet is not really edible. Overall I would stay here again, but mostly if I was with children or had a strange love for the free circus acts.",
    },
    {
        "stars": 3,
        "text": "One of the best massages I have had. Ive gone to manu 5 star resorts yet I still think Seth's massage is just as good as many of them -- yet at a fraction of the cost.\\n\\nBeware however that this is not a 5 star spa experience! Its a small office with tiny massage rooms and VERY thin walls so you can hear people talking, which is annoying.  The thin walls are a big downside to this place. However, Seth seems to understands these limitations so is willing to give discounts and great deals for his massages which makes me ignore the thin walls\\n\\nIf you like deep tissue,  and are willing to deal with a non-chain small biz massage office, dont want to pay alot for a massage. Btw, Seth is a biz owner and a great masseuse so he books fast and it will take a day or two to call you back. Be patient though!  I highly recommend Seth for a great deep tissue massage!!",
    },
    {
        "stars": 0,
        "text": "HORRIBLE! Got a manicure and polish started chipping after just a few hours! The worker that did manicure seemed uninterested and annoyed to be there while rushing through manicure. Overall poor experience. Will never go back",
    },
]

eli5 = [  # https://huggingface.co/docs/transformers/tasks/masked_language_modeling #
    {
        "text": "I did some research as a layman. One forum user [claims it contains](_URL_1_) 300mg of Transfluthrin, but I can't seem to find any SC Johnson or Raid site that corroborates that. Transfluthrin is a pyrethroid, related to a pesticide produced by Chrysanthemums called pyrethrum. The EPA has [scant information on pyrethroids](_URL_0_), but the consensus seems to be that low doses are mostly harmless."
    },
    {
        "text": "While Freud was successful in bringing applied psychology into mainstream awareness and practice, equally we still have some throw-backs to his now-unsupported theories.\n\nOne of those is the intense focus on childhood experience and upbringing including as you said believing that because their parents did X they learnt Y.\n\nIn many respects from a practical point of view knowing the exact learning history of a harmful belief is irrelevant to learning a more helpful one. I would also argue that many people's claims of where their beliefs originated has more to do with their narration of themselves rather than accurate summary about their own cognition.\n\nHOWEVER a caveat - many people may find identifying a potential learning history such as 'my parents did X' may find this makes it easier to change said belief. Without such a rationale people may not even identify what their beliefs are explicitly or acknowledge them as a learnt function (rather than an accurate understanding).\n\nFor example Patient A might believe all people are untrustworthy and abusive, thusly struggling to maintain relationships. Questioning their upbringing and what their parents taught them is not typically a valid way to introduce change of this belief, but believing that comes from how they were raised may illustrate that the belief is changeable rather."
    },
    {
        "text": "Marine biologists and geologists often have opportunities to travel, depending on what kind of employment you can land. Also, while in college you should look into studying abroad."
    },
    {
        "text": 'It\'s related to the "beat" period/frequency - there\'s a wiki article here: _URL_0_ (and also something very relevant here: _URL_1_ )\n\nThis happens if you have two notes that are *slightly* different in frequency. The interferences broduces a volume oscillation with a frequency equal to the difference in frequencies. You can hear this if you\'re trying to tune a guitar. If you\'re off by like 5 Hz, you hear a rapid "wawawawawawawa" as the volume oscillates at 5 Hz. As you bring the string in tune, the beat frequency gets slower and slower ("wawawawaw waaa waaaa waaaaaaaaaa") as the difference in frequencies goes to zero.'
    },
    {
        "text": 'I would give you the exact details, but I actually found a nice little website that explains it all very simply. If you have further questions, I will be happy to answer; but I think the "What\'s Going On?" section answers your question nicely.\n\n_URL_0_'
    },
    {
        "text": "The diamagnetic material will have an induced field in the opposite direction of the exciting material, pushing the field lines somewhat out and away from the wall, depending on the magnetic susceptibility of the material. Superconductors have a magnetic susceptibility of -1 and are perfect diamagnets, so will completely expel the field lines. For a weaker diamagnet (everything else) the field lines will still penetrate the wall but will be attenuated and 'flattened' a bit."
    },
    {
        "text": "Ease of use usually picks the winner, and the resulting decline in usage picks the loser.\n\nIn your example, [*gaol*](_URL_0_) is still used in the UK and Australia, but not nearly as much as *jail*. *Gaol* uses more column width (which has given *jail* the nod in Australian newspapers) and it's also easily misread, usually as *goal*. If the trend continues, eventually everyone will spell it *jail*, and *gaol* will become an archaic spelling. So it is with language."
    },
    {
        "text": "So it's kind of outside the scope of the effect of diet on genetically small ethnic groups - e.g. \"pygmies\"; but the post WW II economic changes in Japan are a classic case of a people who were stereotypically small, but only due to general dietary/protein scarcity. In the japanese economic boom of the late 1950's and 1960s, protein became much more available to the population at large. This had some interesting economic effects: school classrooms and student desks were too small. The school infrastructure had been sized for the steretypical small japanese population; but with the increased availability of dietary proteins during critical childhood development years, they grew to sizes similar to people in other developed economies. I also have some friends who grew up in former communist states in Eastern Europe in the 50's and 60's. Both of them are very small ~ 5' 1\" - 5' 3\". However, emigrating to the US and having children here, their children are very much taller than they are."
    },
    {
        "text": "> f I was given a book in a language I don't understand and get no help understanding it, given unlimited time Could I one day translate it into English?\n\nCould you (or I) do it ? Who knows. It has certainly been done. The best example that comes to my mind is Linear B. The bulk of the work cracking it was done by Michael Ventris, a self taught Linguist. But credit should also be given to Michael Chadwick. (And for the sake of completeness Alice Kober's contribution has been recently regarded as underrecognized.)\n\n_URL_1_\n\nIt's a great story. I thoroughly recommend Michael Chadwick's book, The Decipherment of Linear B:\n\n_URL_0_"
    },
    {
        "text": "I suspect that a room temperature superconductor (if even possible) would contain vortices (pools of non-superconducting material). It wouldn't perfectly super-conduct in all regions and there would be some heat loss, overall entropy would be created. Also, most superconductors can only handle a certain amount of current and will not super-conduct past that point. There's more than just a factor of temperature when it comes to superconducting. You have critical fields, current, and pressure."
    },
    {
        "text": "I am not an expert in this field, but I am a scientist and I read the first paper and looked over the others. Here is my impression:\n\n* The first paper is written in a very unusual fashion. It may be a thesis. I would be very surprised if it has seen peer review.\n\n* The \"operators\" are the researchers themselves. This is not a suitable design to avoid bias and conflicts of interest.\n\n* The experiment was not systematically designed and included only a small dataset. The authors note this themselves, so it's not clear to me why they didn't do a better job.\n\n* The \"anomalies\" that are reported are laughably ridiculous. For example, they report that when the operators wanted the robot to spend longer on the table, the robot did indeed spend longer on the table. They report p=0.02 for this effect and consider this significant (the other papers have similarly unimpressive p-values). This is where the real problems lie. First of all, this is not a particularly low probability. If an equivalent experiment were run 50 times and the null result (no psychokinesis) were true, we would expect to see a result at least this improbable once. We have no clue how many similar experiments were run prior to this publication. With such a simple experimental set-up, it is easy to run a very large number of trials. So why are so few reported? Have some previous experiments been omitted? It is not like a drug trial where the experiment can not be simply repeated as many times as you like. Even for large complex experiments, there are issues surrounding publication bias (i.e., people not publishing results they don't like). Here, where experimental repeats are trivial, alarm bells are very much ringing.\n\nMoreover, p=0.02 would not be considered significant by any Bayesian. Typically p=0.05 is used as a cut-off for statistical significance, but let us not forget that that value is essentially arbitrary. It is a reasonable value for many purposes, but p < 0.05 is not the be all and end all. Your choice of p-value for significance should really be determined by:\n\n(a) Your goal. Is it better to admit false positives or to avoid false negatives? If, for instance, you were trialling a new drug that could replace a drug that you know already works well, you don't want to risk accepting a false positive, so you should choose a more conservative p-value for significance, e.g., p = 0.01.\n\n(b) The prior probability. In other words, what is the burden of proof on the hypothesis that you are testing? If you show me that a drug known to act on sleep promoting pathways increases sleep at night with a p=0.02 chance that the result is a fluke, I'm willing to accept that it is likely correct. If you show me that people who eat a handful of jelly beans immediately gain 65 IQ points relative to those who eat a handful of skittles with a p=0.02 chance that the result is a fluke, I'm very likely going to conclude that the result is a fluke and ask for a more stringent proof of the result.\n\nFinally, the results here are frequently from multiple comparisons, which requires a reassessment of statistical significance. By that, I mean many different analyses were performed on the data and then the most significant ones were held up as impressive. However, every time you perform a new analysis, you are increasing the probability of finding a significant result by chance. That is not addressed by the authors.\n\nIn sum, this is just really really bad science, I'm afraid."
    },
    {
        "text": "Eat, pray, and love...\n\nNo, just kidding.\n\n1. Practice good hygiene\n2. Monitor energy balance (calories in vs. calories out)\n3. Cultivate close relationships and personal interests\n\nModerate your vices, and don't get too caught up in trying to live forever; life is too short."
    },
]


pdexplorer/tests/fixtures.py
pandichef/pdexplorer
import pytest
import altair as alt
from vega_datasets import data
from ..webuse import webuse
from .._altair_mapper import circlechart_  # type: ignore
from .._quietly import quietly


def test_altair():
    source = data.cars()
    dict_repr = (
        alt.Chart(source)
        .mark_circle(size=60)
        .encode(
            x="Horsepower",
            y="Miles_per_Gallon",
            color="Origin",
            tooltip=["Name", "Origin", "Horsepower", "Miles_per_Gallon"],
        )
    ).to_dict()
    assert dict_repr["encoding"]["x"] == {"field": "Horsepower", "type": "quantitative"}


def test_pdexplorer_version():
    with quietly():
        webuse("cars", "vega")
    dict_repr = (
        circlechart_(size=60)
        .encode(
            x="horsepower",
            y="miles_per_gallon",
            color="origin",
            tooltip=["name", "origin", "horsepower", "miles_per_gallon"],
        )
        .to_dict()
    )
    assert dict_repr["mark"] == {"type": "circle", "size": 60}
    assert dict_repr["encoding"]["x"] == {"field": "horsepower", "type": "quantitative"}
    assert dict_repr["encoding"]["y"] == {
        "field": "miles_per_gallon",
        "type": "quantitative",
    }


def test_commandarg():
    with quietly():
        webuse("cars", "vega")
    dict_repr = circlechart_(
        "miles_per_gallon horsepower, color(origin) \
        tooltip(name origin horsepower miles_per_gallon)",
        size=60,
    ).to_dict()
    assert dict_repr["mark"] == {"type": "circle", "size": 60}
    assert dict_repr["encoding"]["x"] == {
        "field": "horsepower",
        "title": "Horsepower",
        "type": "quantitative",
    }
    assert dict_repr["encoding"]["y"] == {
        "field": "miles_per_gallon",
        "title": "Miles_per_Gallon",
        "type": "quantitative",
    }


def test_simple_scatter_plot():
    with quietly():
        webuse("cars", "vega")
    dict_repr = circlechart_("miles_per_gallon horsepower").to_dict()
    assert dict_repr["mark"] == {"type": "circle"}
    # assert dict_repr["mark"] == "circle"
    assert dict_repr["encoding"]["x"] == {
        "field": "horsepower",
        "title": "Horsepower",
        "type": "quantitative",
    }
    assert dict_repr["encoding"]["y"] == {
        "field": "miles_per_gallon",
        "title": "Miles_per_Gallon",
        "type": "quantitative",
    }
    dict_repr = circlechart_("miles_per_gallon horsepower", use_labels=False).to_dict()
    assert dict_repr["encoding"]["x"] == {
        "field": "horsepower",
        "type": "quantitative",
    }
    assert dict_repr["encoding"]["y"] == {
        "field": "miles_per_gallon",
        "type": "quantitative",
    }


def test_stacked():
    with quietly():
        webuse("cars", "vega")
    chartA = circlechart_(
        "miles_per_gallon weight_in_lbs", calculate_variable_labels=False
    )
    chartB = circlechart_("horsepower weight_in_lbs", calculate_variable_labels=False)
    dict_repr = (chartA & chartB).to_dict()
    # "mark": {"type": "circle"},
    # "mark": {"type": "circle"},
    vconcat = [
        {
            "mark": {"type": "circle"},
            "encoding": {
                "x": {
                    "field": "weight_in_lbs",
                    "title": "Weight_in_lbs",
                    "type": "quantitative",
                },
                "y": {
                    "field": "miles_per_gallon",
                    "title": "Miles_per_Gallon",
                    "type": "quantitative",
                },
            },
        },
        {
            "mark": {"type": "circle"},
            "encoding": {
                "x": {
                    "field": "weight_in_lbs",
                    "title": "Weight_in_lbs",
                    "type": "quantitative",
                },
                "y": {
                    "field": "horsepower",
                    "title": "Horsepower",
                    "type": "quantitative",
                },
            },
        },
    ]
    assert dict_repr["vconcat"] == vconcat
    dict_repr = circlechart_(
        "miles_per_gallon horsepower weight_in_lbs",
        stacked=True,
        calculate_variable_labels=False,
    ).to_dict()
    assert dict_repr["vconcat"] == vconcat


def test_stacked_calculate_variable_labels():
    with quietly():
        webuse("cars", "vega")
    chartA = circlechart_(
        "miles_per_gallon weight_in_lbs", calculate_variable_labels=True
    )
    chartB = circlechart_("horsepower weight_in_lbs", calculate_variable_labels=True)
    dict_repr = (chartA & chartB).to_dict()
    # "mark": {"type": "circle"},
    # "mark": {"type": "circle"},
    vconcat = [
        {
            "mark": {"type": "circle"},
            "encoding": {
                "x": {
                    "field": "weight_in_lbs",
                    "title": "Weight_in_lbs",
                    "type": "quantitative",
                },
                "y": {
                    "field": "miles_per_gallon",
                    "title": "Miles_per_Gallon",
                    "type": "quantitative",
                },
            },
            "transform": [
                {"calculate": "datum.name", "as": "Name"},
                {"calculate": "datum.miles_per_gallon", "as": "Miles_per_Gallon"},
                {"calculate": "datum.cylinders", "as": "Cylinders"},
                {"calculate": "datum.displacement", "as": "Displacement"},
                {"calculate": "datum.horsepower", "as": "Horsepower"},
                {"calculate": "datum.weight_in_lbs", "as": "Weight_in_lbs"},
                {"calculate": "datum.acceleration", "as": "Acceleration"},
                {"calculate": "datum.year", "as": "Year"},
                {"calculate": "datum.origin", "as": "Origin"},
            ],
        },
        {
            "mark": {"type": "circle"},
            "encoding": {
                "x": {
                    "field": "weight_in_lbs",
                    "title": "Weight_in_lbs",
                    "type": "quantitative",
                },
                "y": {
                    "field": "horsepower",
                    "title": "Horsepower",
                    "type": "quantitative",
                },
            },
            "transform": [
                {"calculate": "datum.name", "as": "Name"},
                {"calculate": "datum.miles_per_gallon", "as": "Miles_per_Gallon"},
                {"calculate": "datum.cylinders", "as": "Cylinders"},
                {"calculate": "datum.displacement", "as": "Displacement"},
                {"calculate": "datum.horsepower", "as": "Horsepower"},
                {"calculate": "datum.weight_in_lbs", "as": "Weight_in_lbs"},
                {"calculate": "datum.acceleration", "as": "Acceleration"},
                {"calculate": "datum.year", "as": "Year"},
                {"calculate": "datum.origin", "as": "Origin"},
            ],
        },
    ]
    assert dict_repr["vconcat"] == vconcat
    dict_repr = circlechart_(
        "miles_per_gallon horsepower weight_in_lbs",
        stacked=True,
        calculate_variable_labels=True,
    ).to_dict()
    assert dict_repr["vconcat"] == [
        {
            "mark": {"type": "circle"},
            "encoding": {
                "x": {
                    "field": "weight_in_lbs",
                    "title": "Weight_in_lbs",
                    "type": "quantitative",
                },
                "y": {
                    "field": "miles_per_gallon",
                    "title": "Miles_per_Gallon",
                    "type": "quantitative",
                },
            },
        },
        {
            "mark": {"type": "circle"},
            "encoding": {
                "x": {
                    "field": "weight_in_lbs",
                    "title": "Weight_in_lbs",
                    "type": "quantitative",
                },
                "y": {
                    "field": "horsepower",
                    "title": "Horsepower",
                    "type": "quantitative",
                },
            },
        },
    ]


def test_yX_format():
    with quietly():
        webuse("cars", "vega")
    dict_repr = circlechart_(
        "miles_per_gallon horsepower weight_in_lbs",
        yX=True,
        stacked=True,
        calculate_variable_labels=False,
    ).to_dict()
    assert dict_repr["vconcat"] == [
        {
            "mark": {"type": "circle"},
            "encoding": {
                "x": {
                    "field": "horsepower",
                    "title": "Horsepower",
                    "type": "quantitative",
                },
                "y": {
                    "field": "miles_per_gallon",
                    "title": "Miles_per_Gallon",
                    "type": "quantitative",
                },
            },
        },
        {
            "mark": {"type": "circle"},
            "encoding": {
                "x": {
                    "field": "weight_in_lbs",
                    "title": "Weight_in_lbs",
                    "type": "quantitative",
                },
                "y": {
                    "field": "miles_per_gallon",
                    "title": "Miles_per_Gallon",
                    "type": "quantitative",
                },
            },
        },
    ]


def test_not_stacked():
    with quietly():
        webuse("cars", "vega")
    dict_repr = circlechart_("miles_per_gallon horsepower weight_in_lbs").to_dict()
    assert dict_repr["encoding"] == {
        "color": {"field": "key", "type": "nominal"},
        "x": {
            "field": "weight_in_lbs",
            "title": "Weight_in_lbs",
            "type": "quantitative",
        },
        "y": {"field": "value", "type": "quantitative"},
    }


def test_transform_calculate_variable_labels():
    with quietly():
        webuse("cars", "vega")
    dict_repr = (
        circlechart_(calculate_variable_labels=False)
        .transform_calculate_variable_labels()
        .encode(
            y="value:Q", x=alt.X("weight_in_lbs", title="Weight_in_Lbs"), color="key:N"
        )
        .transform_fold(
            ["Miles_per_Gallon", "Horsepower"]  # The variable labels are accessible
        )
        .to_dict()
    )
    assert dict_repr["encoding"] == {
        "color": {"field": "key", "type": "nominal"},
        "x": {
            "field": "weight_in_lbs",
            "title": "Weight_in_Lbs",
            "type": "quantitative",
        },
        "y": {"field": "value", "type": "quantitative"},
    }
    assert dict_repr["transform"] == [
        {"calculate": "datum.name", "as": "Name"},
        {"calculate": "datum.miles_per_gallon", "as": "Miles_per_Gallon"},
        {"calculate": "datum.cylinders", "as": "Cylinders"},
        {"calculate": "datum.displacement", "as": "Displacement"},
        {"calculate": "datum.horsepower", "as": "Horsepower"},
        {"calculate": "datum.weight_in_lbs", "as": "Weight_in_lbs"},
        {"calculate": "datum.acceleration", "as": "Acceleration"},
        {"calculate": "datum.year", "as": "Year"},
        {"calculate": "datum.origin", "as": "Origin"},
        {"fold": ["Miles_per_Gallon", "Horsepower"]},
    ]

pdexplorer/tests/test_charts.py
pandichef/pdexplorer
import math
import pandas as pd
from .._dataset import current
from ..use import use
from ..merge import merge
from ..lis import lis
from .._quietly import quietly
from ..keep import keep
from ..collapse import collapse
from ..webuse import webuse
from ..preserve import preserve


_use_local = True


def test_collapse1():
    # https://www.stata.com/manuals/dmerge.pdf, page 7
    # webuse("iris", "vega")
    with quietly():
        webuse("iris", "vega", use_local=_use_local)
        keep('if species == "setosa"')
        sepallength__mean = current._df.sepallength.mean()
        sepalwidth__mean = current._df.sepalwidth.mean()
        petallength__sum = current._df.petallength.sum()
        petalwidth__sum = current._df.petalwidth.sum()
        webuse("iris", "vega", use_local=_use_local)
        collapse(
            "(mean) sepallength sepalwidth (sum) petallength petalwidth, by(species)"
        )
        keep('if species == "setosa"')
        assert math.isclose(current._df.sepallength[0], sepallength__mean)
        assert math.isclose(current._df.sepalwidth[0], sepalwidth__mean)
        assert math.isclose(current._df.petallength[0], petallength__sum)
        assert math.isclose(current._df.petalwidth[0], petalwidth__sum)


def test_collapse2():  # weights
    with quietly():
        webuse("iris", "vega", use_local=_use_local)
        keep('if species == "setosa"')
        denominator = current._df.petalwidth.sum()
        sepallength__mean = (
            current._df.sepallength * current._df.petalwidth
        ).sum() / denominator
        sepalwidth__mean = (
            current._df.sepalwidth * current._df.petalwidth
        ).sum() / denominator
        webuse("iris", "vega", use_local=_use_local)
        collapse("(mean) sepallength sepalwidth [w=petalwidth], by(species)")
        keep('if species == "setosa"')
        assert math.isclose(current._df.sepallength[0], sepallength__mean)
        assert math.isclose(current._df.sepalwidth[0], sepalwidth__mean)

pdexplorer/tests/test_collapse.py
pandichef/pdexplorer
from .._commandarg import parse, parse_options


def test_parse_options1():
    parsed = parse_options("by(hello world) verbose")
    assert parsed == {"by": "hello world", "verbose": True}


def test_parse_options2():
    parsed = parse_options("   by(hello   world)    verbose")
    assert parsed == {"by": "hello   world", "verbose": True}


def test_parse_options3():
    parsed = parse_options("by(species)")
    assert parsed == {"by": "species"}


def test_parse_options4():
    parsed = parse_options("by(species)", split_values=True)
    assert parsed == {"by": ["species"]}


def test_parse_commandarg1():
    parsed = parse("  brave      huxley   = exp in   new [  w=balance], world")
    assert parsed == {
        "if": None,
        "using": None,
        "=": "exp",
        "in": "new",
        "weight": "w=balance",
        "world": True,
        "options": "world",
        "anything": "brave huxley",
    }


def test_parse_commandarg2():
    parsed = parse('if species == "setosa"')
    assert parsed["if"] == 'species == "setosa"'


def test_parse_commandarg3():
    parsed = parse("(mean) sepallength sepalwidth [w=petalwidth], by(species)")
    assert parsed == {
        "if": None,
        "using": None,
        "=": None,
        "in": None,
        "weight": "w=petalwidth",
        "by": "species",
        "anything": "(mean) sepallength sepalwidth",
        "options": "by(species)",
    }


def test_parse_commandarg4_name_clash():
    parsed = parse(
        '  brave      huxley   = exp if index == 5 and serif == "sans" in   new [  w=balance], world'
    )
    # print(parsed)
    assert parsed == {
        "if": 'index == 5 and serif == "sans"',
        "using": None,
        "=": "exp",
        "in": "new",
        "weight": "w=balance",
        "world": True,
        "anything": "brave huxley",
        "options": "world",
    }


def test_parse_commandarg_exp_only():
    parsed = parse("a = 2")
    # print(parsed)
    assert parsed == {
        "if": None,
        "using": None,
        "=": "2",
        "in": None,
        "weight": None,
        "options": None,
        "anything": "a",
    }


def test_parse_commandarg_if_only():
    parsed = parse('if _merge!="both"')
    # print(parsed)
    assert parsed == {
        "if": '_merge!="both"',
        "using": None,
        "=": None,
        "in": None,
        "weight": None,
        "options": None,
        "anything": None,
    }

pdexplorer/tests/test_commandarg.py
pandichef/pdexplorer
from .._dataset import current
from ..drop import drop
from ..webuse import webuse
from .._quietly import quietly

# from magicpandas.frame import MagicDataFrame
# import warnings

# with warnings.catch_warnings():
#     warnings.simplefilter("ignore")


def test_drop_columns():
    with quietly():
        webuse("auto", use_local=True)
    assert len(current.df.columns) == 12  # type: ignore
    with quietly():
        drop("mak*")  # drop any column that starts with "mak"
    assert len(current.df.columns) == 11  # type: ignore

pdexplorer/tests/test_drop.py
pandichef/pdexplorer
from .._dataset import current
from ..keep import keep
from ..webuse import webuse
from ..drop import dropin
from ..keep import keep
from .._quietly import quietly


def test_dropin1():
    with quietly():
        webuse("auto", use_local=True)
        row_count0 = len(current.df)
        dropin(f"{row_count0 - 5}:")
        row_count1 = len(current.df)
        assert row_count0 - row_count1 == 5


def test_dropin2():
    with quietly():
        webuse("auto", use_local=True)
        keep("if price > 5000")
        row_count0 = len(current.df)
        dropin(f"{row_count0 - 5}:")
        row_count1 = len(current.df)
        assert row_count0 - row_count1 == 5
    # print(row_count0)

pdexplorer/tests/test_dropin.py
pandichef/pdexplorer
from math import isclose
import pandas as pd
from .._dataset import current
from ..use import use
from ..merge import merge
from ..lis import lis
from .._quietly import quietly
from ..keep import keep
from ..collapse import collapse
from ..webuse import webuse
from ..preserve import preserve
from ..egen import egen
from ..save import save


def test_egen1():
    with quietly():
        webuse("iris", "vega", use_local=True)
        egen("asdf = sum(  petalwidth)", by="species")
        keep('if species == "setosa"')
        egen_calculation = current.df.asdf[0]
        pandas_calculation = current.df.petalwidth.mean()
        isclose(egen_calculation, pandas_calculation)


def test_egen2():
    with quietly():
        webuse("iris", "vega", use_local=True)
        egen("count = count(petalwidth)", by="species")
        isclose(current.df["count"][0], 50)
        egen("median = median(petalwidth)", by="species")
        isclose(current.df["median"][0], 0.2)
        egen("min = min(petalwidth)", by="species")
        isclose(current.df["min"][0], 0.1)
        egen("max = max(petalwidth)", by="species")
        isclose(current.df["max"][0], 0.6)


def test_egen3():
    with quietly():
        webuse("iris", "vega", use_local=True)
        egen("rank = rank(petalwidth)", by="species")
        isclose(current.df["rank"][0], 20.0)

pdexplorer/tests/test_egen.py
pandichef/pdexplorer
import pandas as pd
from ..fine_tuning.ftgpt import (
    _convert_tabular_format_to_openai_format,
    _write_records_to_stringio,
)

# Data in JSON-like format
data = [
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also sarcastic.",
            },
            {"role": "user", "content": "What's the capital of France?"},
            {
                "role": "assistant",
                "content": "Paris, as if everyone doesn't know that already.",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also sarcastic.",
            },
            {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"},
            {
                "role": "assistant",
                "content": "Oh, just some guy named William Shakespeare. Ever heard of him?",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also sarcastic.",
            },
            {"role": "user", "content": "How far is the Moon from Earth?"},
            {
                "role": "assistant",
                "content": "Around 384,400 kilometers. Give or take a few, like that really matters.",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also pessimistic.",
            },
            {"role": "user", "content": "What's the capital of France?"},
            {
                "role": "assistant",
                "content": "What's the point of knowing the capital of France when the world is falling apart?",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also pessimistic.",
            },
            {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"},
            {
                "role": "assistant",
                "content": "William Shakespeare wrote 'Romeo and Juliet', but what does it matter? The story ends in tragedy anyway.",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also pessimistic.",
            },
            {"role": "user", "content": "How far is the Moon from Earth?"},
            {
                "role": "assistant",
                "content": "The Moon is about 384,400 kilometers away from Earth, not that it would make any difference to your life.",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also empathetic.",
            },
            {"role": "user", "content": "What's the capital of France?"},
            {
                "role": "assistant",
                "content": "I understand why you'd want to know the capital of France. It's Paris.",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also empathetic.",
            },
            {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"},
            {
                "role": "assistant",
                "content": "Ah, 'Romeo and Juliet', a tale of love and woe. It was written by William Shakespeare.",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also empathetic.",
            },
            {"role": "user", "content": "How far is the Moon from Earth?"},
            {
                "role": "assistant",
                "content": "The Moon is roughly 384,400 kilometers away from Earth. It's fascinating how we're connected to something so far away, isn't it?",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also formal.",
            },
            {"role": "user", "content": "What's the capital of France?"},
            {
                "role": "assistant",
                "content": "The capital of the French Republic is Paris.",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also formal.",
            },
            {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"},
            {
                "role": "assistant",
                "content": "The play 'Romeo and Juliet' was penned by William Shakespeare, an eminent English playwright.",
            },
        ]
    },
    {
        "messages": [
            {
                "role": "system",
                "content": "Marv is a factual chatbot that is also formal.",
            },
            {"role": "user", "content": "How far is the Moon from Earth?"},
            {
                "role": "assistant",
                "content": "The average distance from the Earth to the Moon is approximately 384,400 kilometers.",
            },
        ]
    },
]


# Create empty lists for each role
system_msgs = []
user_msgs = []
assistant_msgs = []

# Iterate through the data to populate the lists
for entry in data:
    for msg in entry["messages"]:
        role = msg["role"]
        content = msg["content"]
        if role == "system":
            system_msgs.append(content)
        elif role == "user":
            user_msgs.append(content)
        elif role == "assistant":
            assistant_msgs.append(content)

# Create a pandas DataFrame in tabular format
# Note that OpenAI requires at least 10 examples
df = pd.DataFrame(
    {"system": system_msgs, "user": user_msgs, "assistant": assistant_msgs}
)


def test_convert_format():
    assert _convert_tabular_format_to_openai_format(df[:3]) == data[:3]


def test_write_to_stringio():
    stringio_object = _write_records_to_stringio(data[:3])
    with stringio_object as f:
        stringio_content = f.read()
    correct_content = """{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]}
{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]}
{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]}
"""
    assert stringio_content == correct_content

pdexplorer/tests/test_ftgpt.py
pandichef/pdexplorer
import os
import subprocess
import pytest
import sys
import pandas as pd
from ..webuse import webuse
from ..use import use
from .._dataset import current

# from ..nn.finetune import finetune
# from ..fine_tuning.fthuggingface import fthuggingface, fthuggingface_old, askhuggingface
from ..fine_tuning.fthuggingface import fthuggingface, askhuggingface
from ..nn.pipeline import pipeline


# def df_maker(records):
#     y0 = pd.DataFrame.from_records(records)
#     y0["split"] = "train"
#     y1 = pd.DataFrame.from_records(records)
#     y1["split"] = "test"

#     return pd.concat([y0, y1], axis=0)


# @pytest.mark.skip
# @pytest.mark.skipif(sys.platform != "win32", reason="only run locally")
@pytest.mark.slow
def test_finetune_text_generation():
    from .fixtures import eli5

    df = pd.DataFrame.from_records(eli5)
    # eli_df = df_maker(eli5)
    use(df)
    fthuggingface("text", "text-generation")  # , num_examples=100)
    assert "text" in current.df.columns
    askhuggingface("This is my first Yelp review.", "text-generation")
    subprocess.run(f"rm -rf {current.last_huggingface_ftmodel_dir}", shell=True)
    # os.system(f"rm -rf {current.last_huggingface_ftmodel_dir}")


# @pytest.mark.skip
@pytest.mark.slow
@pytest.mark.skipif(sys.platform != "win32", reason="only run locally")
def test_finetune_fill_mask():
    from .fixtures import eli5

    df = pd.DataFrame.from_records(eli5)
    # eli_df = df_maker(eli5)
    use(df)
    fthuggingface("text", "fill-mask")  # , num_examples=100)
    assert "text" in current.df.columns
    askhuggingface("This is my first <mask> review.", "fill-mask")
    subprocess.run(f"rm -rf {current.last_huggingface_ftmodel_dir}", shell=True)
    # os.system(f"rm -rf {current.last_huggingface_ftmodel_dir}")


# @pytest.mark.skip
@pytest.mark.slow
@pytest.mark.skipif(sys.platform != "win32", reason="only run locally")
def test_finetune_sentiment_analysis():
    from .fixtures import yelp_reviews

    # yelp_reviews_df = df_maker(yelp_reviews)
    # use(yelp_reviews_df)
    df = pd.DataFrame.from_records(yelp_reviews)
    use(df)
    fthuggingface("stars text", "sentiment-analysis")  # , num_examples=100)
    assert "stars" in current.df.columns
    askhuggingface("This is my first Yelp review.", "sentiment-analysis")
    subprocess.run(f"rm -rf {current.last_huggingface_ftmodel_dir}", shell=True)
    # os.system(f"rm -rf {current.last_huggingface_ftmodel_dir}")


# @pytest.mark.skip
@pytest.mark.slow
@pytest.mark.skipif(sys.platform != "win32", reason="only run locally")
def test_finetune_text_classification():
    from .fixtures import yelp_reviews

    # yelp_reviews_df = df_maker(yelp_reviews)
    df = pd.DataFrame.from_records(yelp_reviews)
    use(df)
    fthuggingface("stars text", "text-classification")
    assert "stars" in current.df.columns
    askhuggingface("This is my first Yelp review.", "text-classification")
    subprocess.run(f"rm -rf {current.last_huggingface_ftmodel_dir}", shell=True)

pdexplorer/tests/test_fthuggingface.py
pandichef/pdexplorer
def test_from_pdexplorer_import_all():
    from pdexplorer import drop, regress

pdexplorer/tests/test_import.py
pandichef/pdexplorer
from .._dataset import current
from ..keep import keep
from ..webuse import webuse
from .._quietly import quietly

# from magicpandas.frame import MagicDataFrame
# import warnings

# with warnings.catch_warnings():
#     warnings.simplefilter("ignore")
#     import statsmodels.api as sm


def test_keep_columns():
    # singleton.df = sm.datasets.get_rdataset("Duncan", "carData").data
    with quietly():
        webuse("auto", use_local=True)
    assert len(current._df.columns) == 12  # type: ignore
    with quietly():
        keep("mak*")  # keep any column that starts with "mak"
    assert len(current._df.columns) == 1  # type: ignore

pdexplorer/tests/test_keep.py
pandichef/pdexplorer
from .._dataset import current
from ..keep import keep
from ..webuse import webuse
from ..keep import keepin

# from ..keepif import keepif
from .._quietly import quietly


def test_keepin1():
    with quietly():
        webuse("auto", use_local=True)
        row_count0 = len(current.df)
        keepin(f"{row_count0 - 5}:")
        row_count1 = len(current.df)
        assert row_count1 == 5

pdexplorer/tests/test_keepin.py
pandichef/pdexplorer
import pandas as pd
from .._dataset import current
from ..use import use
from ..merge import merge
from ..lis import lis
from .._quietly import quietly
from ..keep import keep
from ..preserve import preserve


def test_merge1():
    # https://www.stata.com/manuals/dmerge.pdf, page 7
    using = pd.DataFrame({"id": [1, 2, 4], "wgt": [130, 180, 110],})
    master = pd.DataFrame({"id": [1, 2, 5], "age": [22, 56, 17],})
    with quietly():
        use(using)
        preserve()
        use(master)
        merge("id", how="outer")
        # res = master.merge(using, how="outer", on="id", indicator=True)
        # print(res)
        assert len(current.df) == 4
        keep('if _merge == "both"')
        assert len(current.df) == 2


def test_merge2():
    # https://www.stata.com/manuals/dmerge.pdf, page 7
    using = pd.DataFrame({"id": [1, 2, 4], "wgt": [130, 180, 110],})
    master = pd.DataFrame({"id": [1, 2, 5], "age": [22, 56, 17],})
    with quietly():
        use(using)
        preserve()
        use(master)
        merge("id")
        assert len(current.df) == 3

pdexplorer/tests/test_merge.py
pandichef/pdexplorer
from math import isclose
import pandas as pd
from ..finance.mtgprice import mtgprice
from ..finance.mtgyield import mtgyield
from ..use import use
from .._quietly import quietly
from .._dataset import current
from ..drop import drop

loan_df = {
    "beyield": {0: 6, 1: 7},
    "coupon": {0: 5, 1: 5},
    "cpr": {0: 15, 1: 15},
    "age": {0: 0, 1: 0},
    "origfixedterm": {0: 360, 1: 360},
    "origterm": {0: 360, 1: 360},
    "servicing_fee": {0: 0, 1: 0},
}


def test_mtg_price_yield():
    with quietly():
        use(pd.DataFrame(loan_df))
        yield_series = current.df.beyield.copy()
        mtgprice()
        drop("beyield")
        assert "beyield" not in current.df.columns
        mtgyield()
        yield_series2 = current.df.beyield.copy()
        assert isclose(yield_series[0], yield_series2[0], abs_tol=0.000001)
        assert isclose(yield_series[1], yield_series2[1], abs_tol=0.000001)

pdexplorer/tests/test_mtg_price_yield.py
pandichef/pdexplorer
import sys
from io import StringIO
from math import isclose
import pytest
from ..webuse import webuse
from .._dataset import current
from .._quietly import quietly, quietly_decorator
from ..regress import regress


def rprint():
    print("rprinted")


@pytest.mark.skipif(sys.platform != "win32", reason="only run locally")
def test_quietly_hard():
    with quietly(hard=True) as captured_output:
        webuse("auto", use_local=True)
        rprint()
    s = captured_output.getvalue()
    assert s.startswith("(1978 Automobile Data)") and s.endswith("rprinted\n")
    s2 = current.captured_output.getvalue()
    assert s2.startswith("(1978 Automobile Data)") and s2.endswith("rprinted\n")


@pytest.mark.skipif(sys.platform != "win32", reason="only run locally")
def test_quietly_soft():
    with quietly(hard=False) as captured_output:
        webuse("auto", use_local=True)
        rprint()
    s = captured_output.getvalue()
    assert s.startswith("(1978 Automobile Data)") and not s.endswith("rprinted\n")
    s2 = current.captured_output.getvalue()
    assert s2.startswith("(1978 Automobile Data)") and not s2.endswith("rprinted\n")


@quietly_decorator(hard=True)
def fnc_to_wrap_hard():
    webuse("auto", use_local=True)
    rprint()


@quietly_decorator(hard=False)
def fnc_to_wrap_soft():
    webuse("auto", use_local=True)
    rprint()


@pytest.mark.skipif(sys.platform != "win32", reason="only run locally")
def test_quietly_decorator_hard():
    fnc_to_wrap_hard()
    s2 = current.captured_output.getvalue()
    assert s2.startswith("(1978 Automobile Data)") and s2.endswith("rprinted\n")


@pytest.mark.skipif(sys.platform != "win32", reason="only run locally")
def test_quietly_decorator_soft():
    fnc_to_wrap_soft()
    s2 = current.captured_output.getvalue()
    assert s2.startswith("(1978 Automobile Data)") and not s2.endswith("rprinted\n")

pdexplorer/tests/test_quietly.py
pandichef/pdexplorer
from math import isclose
import pandas as pd
import pytest
from ..webuse import webuse
from .._dataset import current
from .._quietly import quietly
from ..regress import regress
from ..returnlist import e_
from ..predict import predict


def test_regress_smf_vs_sm():
    # https://www.statsmodels.org/dev/generated/statsmodels.regression.linear_model.OLS.html
    # compare sm and smf
    import statsmodels.api as sm

    with quietly():
        webuse("Duncan__carData", "rdatasets", use_local=True)
    Y = current.df["income"]
    X = current.df["education"]
    X = sm.add_constant(X)
    model = sm.OLS(Y, X)
    results = model.fit()
    with quietly():
        results2 = regress("income education")

    assert isclose(results.params["const"], results2.params["Intercept"])  # type:ignore
    assert isclose(
        results.params["education"], results2.params["education"]  # type: ignore
    )


def test_ereturn_list():
    webuse("auto", use_local=True)
    res = regress("price mpg")
    assert type(res.nobs) == float  # type: ignore
    assert type(e_("N")) == int
    assert int(res.nobs) == e_("N")  # type: ignore


def test_predict():
    webuse("auto", use_local=True)
    current.df = current.df[:10]  # use first 10 observations #
    regress("price mpg weight")
    predict("newvar")
    s = current.df["newvar"]
    s2 = pd.Series(  # results from Stata #
        [
            4717.105,
            5382.828,
            3682.33,
            5525.76,
            7654.441,
            6691.233,
            2885.706,
            5632.806,
            7107.386,
            5894.405,
        ]
    )
    assert (abs(s2 - s) < 0.001).all()


# def test_value_labels():
#     webuse("auto", use_local=False)
#     assert False, current.df


def test_predict_with_if():
    webuse("auto", use_local=True)
    # assert False, current.df
    regress('price mpg weight if foreign=="Foreign"')
    predict("newvar")
    s = current.df["newvar"]
    s2 = pd.Series(  # results from Stata #
        [
            9605.675,
            11870.015,
            8110.481,
            11295.099,
            15673.335,
            13500.108,
            5917.476,
            11449.774,
            14622.389,
            12088.253,
            16982.073,
            14765.061,
            16637.398,
            5239.443,
            13642.779,
            10894.635,
            11100.869,
            8638.069,
            12242.928,
            5271.224,
            13139.199,
            13178.753,
            13880.794,
            3660.909,
            8181.816,
            19651.107,
            19032.406,
            14404.152,
            7801.13,
            15589.995,
            13817.232,
            11953.355,
            15950.904,
            9129.645,
            15451.554,
            11624.227,
            11572.669,
            13603.224,
            11933.578,
            8534.952,
            15395.765,
            11188.439,
            3542.245,
            5782.578,
            7412.67,
            11747.121,
            13654.783,
            12468.939,
            11108.643,
            11057.084,
            12191.37,
            8328.718,
            9188.977,
            5151.873,
            8102.707,
            6698.626,
            4656.753,
            6214.823,
            8697.401,
            5500.778,
            5988.812,
            3454.675,
            4549.406,
            12290.256,
            3855.139,
            4811.428,
            7003.746,
            5663.914,
            8344.265,
            5615.899,
            4641.205,
            4390.501,
            4699.851,
            10941.964,
        ]
    )
    assert (abs(s2 - s) < 0.001).all()

pdexplorer/tests/test_regress.py
pandichef/pdexplorer
from math import isclose
import pytest
from ..webuse import webuse
from .._dataset import current
from .._quietly import quietly

from ..regress import regress
from ..ml.regressml import regressml
from ..predict import predict
import pandas as pd


def test_regress_smf_vs_sklearn():
    # compare smf and sklearn
    with quietly():
        webuse("Duncan__carData", "rdatasets", use_local=True)
    with quietly():
        results = regress("income education")
    with quietly():
        results2 = regressml("income education")  # , library="sklearn")
    assert isclose(results.params["Intercept"], results2.intercept_)  # type:ignore
    assert isclose(results.params["education"], results2.coef_[0])  # type:ignore


def test_regress_smf_vs_sklearn_2():
    # compare smf and sklearn
    with quietly():
        webuse("auto", use_local=True)
    with quietly():
        results = regress("price mpg weight")
    with quietly():
        results2 = regressml("price mpg weight")  # , library="sklearn")
    assert isclose(results.params["Intercept"], results2.intercept_)  # type:ignore
    assert isclose(results.params["mpg"], results2.coef_[0])  # type:ignore
    assert isclose(results.params["weight"], results2.coef_[1])  # type:ignore


def test_predict():
    webuse("auto", use_local=True)
    current.df = current.df[:10]  # use first 10 observations #
    regressml("price mpg weight")
    predict("newvar")
    s = current.df["newvar"]
    s2 = pd.Series(  # results from Stata #
        [
            4717.105,
            5382.828,
            3682.33,
            5525.76,
            7654.441,
            6691.233,
            2885.706,
            5632.806,
            7107.386,
            5894.405,
        ]
    )
    assert (abs(s2 - s) < 0.001).all()

pdexplorer/tests/test_regressml.py
pandichef/pdexplorer
from math import isclose
import pytest
from ..webuse import webuse
from .._dataset import current
from .._quietly import quietly

from ..regress import regress
from ..nn.regressnn import regressnn


# def test_regress_smf_vs_pytorch_Rprop():
#     with quietly():
#         webuse("Duncan__carData", "rdatasets", use_local=True)
#     with quietly():
#         results = regress("income education")
#     with quietly():
#         model = regress("income education", library="pytorch", epochs=50)
#     rel_tol = 1  # high tolerance for speed
#     assert isclose(
#         results.params["Intercept"],
#         float(list(model.parameters())[1]),
#         rel_tol=rel_tol,  # type:ignore
#     )
#     assert isclose(
#         results.params["education"],
#         float(list(model.parameters())[0]),
#         rel_tol=rel_tol,  # type:ignore
#     )


# @pytest.mark.skip
@pytest.mark.slow
def test_regress_smf_vs_pytorch_Rprop_2_covariates():
    with quietly():
        webuse("Duncan__carData", "rdatasets", use_local=True)
    # with quietly():
    results = regress("income education prestige")
    with quietly():
        model = regressnn(
            "income education prestige", use_dataloader=True,
        )  # , library="pytorch", epochs=100)
    abs_tol = 0.5
    # print(list(model.parameters()))
    assert isclose(
        results.params["Intercept"],  # type:ignore
        float(list(model.parameters())[1]),  # type:ignore
        abs_tol=abs_tol,  # type:ignore
    )
    # print(list(model.parameters())[0][0][0])
    assert isclose(
        results.params["education"],  # type:ignore
        float(list(model.parameters())[0][0][0]),  # type:ignore
        abs_tol=abs_tol,  # type:ignore
    )
    assert isclose(
        results.params["prestige"],  # type:ignore
        float(list(model.parameters())[0][0][1]),  # type:ignore
        abs_tol=abs_tol,  # type:ignore
    )

pdexplorer/tests/test_regressnn.py
pandichef/pdexplorer
import pandas as pd
from .._dataset import current
from ..use import use
from ..merge import merge
from ..lis import lis
from .._quietly import quietly

# from ..keepif import keepif
from ..reshape import reshapelong, reshapewide
from ..save import save
from ..xpose import xpose
from pandas.testing import assert_frame_equal

long = pd.DataFrame(
    {"i": [1, 1, 2, 2], "j": [1, 2, 1, 2], "stub": [4.1, 4.5, 3.3, 3.0]}
)
wide = pd.DataFrame({"i": [1, 2], "stub1": [4.1, 3.3], "stub2": [4.5, 3.0]})


def test_reshape1():
    # https://www.stata.com/manuals/dmerge.pdf, page 7
    with quietly():
        use(long)
        reshapewide("stub", "i", "j")
    assert_frame_equal(current.df, wide)


def test_reshape2():
    # https://www.stata.com/manuals/dmerge.pdf, page 7
    with quietly():
        use(wide)
        reshapelong("stub", "i", "j")
    assert_frame_equal(current.df, long)

pdexplorer/tests/test_reshape.py
pandichef/pdexplorer
import pandas as pd
from .._dataset import current
from ..use import use
from ..merge import merge
from ..lis import lis
from .._quietly import quietly

# from ..keepif import keepif
from ..reshape import reshapelong, reshapewide
from ..save import save
from ..xpose import xpose
from pandas.testing import assert_frame_equal
from ..sysuse import sysuse
from ..sample import sample


def test_sample_percentage():
    sysuse("nlswork")
    assert len(current.df) == 28534
    counts = current.df["race"].value_counts()
    assert counts["White"] == 20180
    sample(10, random_state=0)
    assert len(current.df) == 2853
    counts = current.df["race"].value_counts()
    assert counts["White"] == 2008


def test_sample_percentage_and_groupby():
    sysuse("nlswork")
    sample(10, by="race", random_state=0)
    assert len(current.df) == 2853
    counts = current.df["race"].value_counts()
    assert counts["White"] == 2018


def test_sample_count():
    sysuse("nlswork")
    sample(100, count=True, random_state=0)
    assert len(current.df) == 100
    counts = current.df["race"].value_counts()
    assert counts["White"] == 60


def test_sample_count_and_groupby():
    sysuse("nlswork")
    sample(100, by="race", count=True, random_state=0)
    assert len(current.df) == 300
    counts = current.df["race"].value_counts()
    assert counts["White"] == 100

pdexplorer/tests/test_sample.py
pandichef/pdexplorer
from .fixtures import df1 as df
from .._search import _search_by_ranges


def test_basic_example():
    # Note: Multiple range (e.g., "rate-origterm age-upb" do not work
    assert _search_by_ranges(df.columns, "origterm-age") == [
        "origterm",
        "origltv",
        "origfico",
        "age",
    ]


def test_two_arguments():
    assert _search_by_ranges(df.columns, "rate-origterm age-upb") == [
        "rate",
        "origfixedterm",
        "origterm",
        "age",
        "origupb",
        "upb",
    ]

pdexplorer/tests/test_search_by_ranges.py
pandichef/pdexplorer
from .fixtures import df1 as df
from .._search import _search_by_wildcard


def test_basic_example():
    assert _search_by_wildcard(df.columns, "orig*") == [
        "origfixedterm",
        "origterm",
        "origltv",
        "origfico",
        "origupb",
    ]


def test_two_arguments():
    assert _search_by_wildcard(df.columns, "orig* *upb") == [
        "origfixedterm",
        "origterm",
        "origltv",
        "origfico",
        "origupb",
        "upb",
    ]

pdexplorer/tests/test_search_by_wildcard.py
pandichef/pdexplorer
from .fixtures import df1 as df
from .._search import search_iterable


def test_wildcard() -> None:
    assert search_iterable(df.columns, "orig* *upb") == [
        "origfixedterm",
        "origterm",
        "origltv",
        "origfico",
        "origupb",
        "upb",
    ]


def test_range() -> None:
    assert search_iterable(df.columns, "origterm-age") == [
        "origterm",
        "origltv",
        "origfico",
        "age",
    ]


def test_range_two_arguments() -> None:
    assert search_iterable(df.columns, "rate-origterm age-upb") == [
        "rate",
        "origfixedterm",
        "origterm",
        "age",
        "origupb",
        "upb",
    ]


def test_lazily() -> None:
    assert search_iterable(df.columns, "origfix origfic") == [
        "origfixedterm",
        "origfico",
    ]


def test_mixed() -> None:
    assert search_iterable(df.columns, "rate ag origterm-origfico *upb") == [
        "rate",
        "age",
        "origterm",
        "origltv",
        "origfico",
        "origupb",
        "upb",
    ]


def test_subtract() -> None:
    assert search_iterable(
        df.columns, "rate ag origterm-origfico *upb", subtract=True
    ) == ["origfixedterm"]

pdexplorer/tests/test_search_iterable.py
pandichef/pdexplorer
from .fixtures import df1 as df
from .._search import _search_lazily


def test_basic_example():
    # unique matches work
    assert _search_lazily(df.columns, "origfix origfic") == [
        "origfixedterm",
        "origfico",
    ]


def test_exact_match():
    assert _search_lazily(df.columns, "origfixedterm origfico") == [
        "origfixedterm",
        "origfico",
    ]


def test_nameerror_ambiguous():
    # with pytest.raises(NameError, message='asdf') as excinfo:
    try:
        _search_lazily(df.columns, "orig")
    except Exception as e:
        assert type(e) == NameError
        assert str(e) == "orig ambiguous abbreviation"


def test_nameerror_var_not_found():
    # with pytest.raises(NameError, message='asdf') as excinfo:
    try:
        _search_lazily(df.columns, "hello")
    except Exception as e:
        assert type(e) == NameError
        assert str(e) == "variable hello not found"

pdexplorer/tests/test_search_lazily.py
pandichef/pdexplorer
from ..plus.srecode import srecode
from .._quietly import quietly
from ..webuse import webuse
from ..keep import keep
from ..tabulate import tabulate
from .._dataset import current


def test_srecode():
    with quietly():
        webuse("auto")
        keep("price")
        srecode("pcat", "price", 4000)
        # tabulate('pcat')
        counts = current.df.pcat.value_counts()
        assert counts["0.00 - 4000.00"] == 11
        assert counts["4000.01 - 8000.00"] == 49
        assert counts["8000.01 - 12000.00"] == 9
        assert counts["12000.01 - 16000.00"] == 5

pdexplorer/tests/test_srecode.py
pandichef/pdexplorer
from ..ml.treeclassify import treeclassify
from .._quietly import quietly
from ..webuse import webuse
from ..keep import keep
from ..tabulate import tabulate
from .._dataset import current


def test_treeclassify_1():
    with quietly():
        webuse("iris", "sklearn", use_local=True)
        # keep("price")
        clf = treeclassify(
            "target sepal* petal*",
            max_depth=2,
            show_graphviz=False,
            use_regressor=False,
        )
        assert clf.predict([[1, 1, 1, 1]])[0] in ["setosa", "versicolor", "virginica"]

pdexplorer/tests/test_treeclassify.py
pandichef/pdexplorer
from typing import Tuple, Dict

# import statsmodels.api as sm
from ._dataset import current, Dataset

# import requests
from urllib.parse import urljoin

# import pyreadstat
# import tempfile
import os
from pandas.io.stata import StataReader
import pandas as pd
from .preserve import preserve
from ._print import _print


# def _use(file_path=None, html_index=0) -> Tuple[pd.DataFrame, Dict]:
def _use(file_path=None) -> Tuple[pd.DataFrame, Dict]:
    """Version of use that returns a """
    # use(): read from clipboard
    if not isinstance(file_path, pd.DataFrame) and not file_path:
        return pd.read_clipboard(), current.METADATA_DEFAULT
    elif not isinstance(file_path, str):  # i.e., it's a DataFrame
        return file_path.copy(), current.METADATA_DEFAULT
    # elif file_path.startswith("http"):
    #     return pd.read_html(file_path)[html_index], current.METADATA_DEFAULT
    elif os.path.isdir(file_path):
        import datasets  # huggingface

        return datasets.load_from_disk(file_path).to_pandas(), current.METADATA_DEFAULT  # type: ignore
    else:
        file_extension = os.path.splitext(file_path)[1]
        if file_extension == ".dta":
            _df_to_return = pd.read_stata(file_path, convert_categoricals=True)
            _metadata_to_return = {}
            with StataReader(file_path) as reader:
                _metadata_to_return["data_label"] = reader.data_label
                _metadata_to_return["variable_labels"] = reader.variable_labels()
                _metadata_to_return["value_labels"] = reader.value_labels()
            return _df_to_return, _metadata_to_return
        elif file_extension == ".csv":
            return pd.read_csv(file_path), current.METADATA_DEFAULT
        elif file_extension in [".xlsx", ".xls"]:
            return pd.read_csv(file_path), current.METADATA_DEFAULT
        # elif file_extension in [".pkl"]:
        #     obj: Dataset = pd.read_pickle(file_path)
        #     return pd.read_csv(file_path), obj.metadata
        else:
            raise TypeError("Invalid file path.")
            # return pd.DataFrame(), {}


# def use(file_path=None, preserve_=True, html_index=0) -> None:
def use(file_path=None, preserve_=False, suppress_print=False) -> None:
    # global current

    if preserve_:
        preserve()

    if isinstance(file_path, str) and os.path.splitext(file_path)[1] == ".pkl":
        import pickle

        with open(file_path, "rb") as file:
            # Deserialize and load the object from the file
            _current = pickle.load(file)
        current.df = _current.df
        current.metadata = _current.metadata
        current.last_openai_ftjob_id = _current.last_openai_ftjob_id
    else:
        current.df, current.metadata = _use(file_path=file_path)

    if not suppress_print:
        _print(current.df)

pdexplorer/use.py
pandichef/pdexplorer
import os
import functools
import tempfile
from urllib.parse import urljoin
import requests
import pandas as pd
from pandas.io.stata import StataReader
from ._dataset import current
from .preserve import preserve
from .use import use
from .save import save
from ._print import _print
from ._quietly import quietly
from .clear import clear, clearall


def cache_file_fetch(webuse_func):
    @functools.wraps(webuse_func)
    def decorator(use_local=False):
        # print('use_local: ', use_local)

        def wrapper(name, *args, **kwargs):
            if use_local:

                def get_local(name, *args, **kwargs):
                    source = webuse_func.__name__.split("_")[-1]
                    try:
                        use(
                            f"{str(name).replace('/','_')}__{source}.dta",
                            suppress_print=True,
                        )
                    except FileNotFoundError:
                        print("File not found locally.  Retrieving from server.")
                        webuse_func(name, *args, **kwargs)
                        file_path = f"{str(name).replace('/','_')}__{source}.dta"
                        save(file_path)
                        # print(current.df.foreign)
                        # use(file_path)
                        print(f"Saving to {file_path}.")

                get_local(name, *args, **kwargs)
            else:
                webuse_func(name, *args, **kwargs)

        return wrapper

    return decorator


def make_metadata_value_labels(df):
    metadata_value_labels = {}
    for catcol in df.select_dtypes(include=["category"]).columns:
        this_label = dict(zip(df[catcol].cat.codes, df[catcol].astype(str)))
        metadata_value_labels.update({catcol: this_label})
    return metadata_value_labels


@cache_file_fetch
def _webuse_stata(
    name="auto", baseurl="https://www.stata-press.com/data/r18/",
):
    # For whatever reason, NamedTemporaryFile wasn't working
    # Note that statsmodels itself has an implementation of webuse, but it doesn't grab the metadata
    url_to_datafile = urljoin(baseurl, name + ".dta")
    response = requests.get(url_to_datafile)

    if response.status_code == 200:
        # Save the downloaded data to a local file

        temp_dir = tempfile.gettempdir()
        temp_file_name = os.path.join(temp_dir, "tmpqbxcec2k.dta")

        with open(temp_file_name, "wb") as temp_file:
            temp_file.write(response.content)
        # singleton.df, singleton.metadata = pyreadstat.read_dta(temp_file_name)
        with StataReader(temp_file_name) as reader:
            current.metadata["data_label"] = reader.data_label
            current.metadata["variable_labels"] = reader.variable_labels()
            current.df = reader.read(convert_categoricals=True)
        # reader.value_labels() doesn't work correctly because Stata #
        # uses a value_label name whereas pandas does not #

        current.metadata["value_labels"] = make_metadata_value_labels(current.df)

        # current.df = pd.read_stata(temp_file_name, convert_categoricals=True)
        os.remove(temp_file_name)
    else:
        raise Exception("Data not found.")


@cache_file_fetch
def _webuse_vega(name="cars"):
    clear()
    from vega_datasets import data

    current.df = eval(f"data.{name}()")
    current.metadata["value_labels"] = make_metadata_value_labels(current.df)


@cache_file_fetch
def _webuse_seaborn(name="tips"):
    clear()
    import seaborn as sns

    current.df = eval(f"sns.load_dataset('{name}')")
    current.metadata["value_labels"] = make_metadata_value_labels(current.df)


@cache_file_fetch
def _webuse_rdataset(name="AirPassengers"):
    clear()
    import statsmodels.api as sm

    name_split = str(name).split("__")
    if len(name_split) == 1:
        name_split.append("datasets")  # This is the default in Rdatasets
    current.df = eval(
        f"sm.datasets.get_rdataset('{name_split[0]}', '{name_split[1]}').data"
    )
    current.metadata["value_labels"] = make_metadata_value_labels(current.df)


@cache_file_fetch
def _webuse_statsmodels(name="longley"):
    clear()
    import statsmodels.api as sm

    current.df = eval(f"sm.datasets.{name}.load_pandas().data")
    current.metadata["value_labels"] = make_metadata_value_labels(current.df)


@cache_file_fetch
def _webuse_huggingface(name="imdb"):
    clear()
    from datasets import load_dataset_builder
    from datasets import load_dataset

    ds_builder = load_dataset_builder(name)
    current.metadata["data_label"] = ds_builder.info.description
    dataset = load_dataset(name)
    _df_all_splits = pd.DataFrame()
    for split_name, split_data in dataset.data.items():
        _df_this_split = split_data.to_pandas()
        _df_this_split["split"] = split_name
        _df_all_splits = pd.concat([_df_all_splits, _df_this_split], ignore_index=False)
    current.df = _df_all_splits
    current.metadata["data_label"] = name
    current.metadata["value_labels"] = make_metadata_value_labels(current.df)


@cache_file_fetch
def _webuse_sklearn(name="iris"):
    clear()
    exec(f"from sklearn.datasets import load_{name}")
    data = eval(f"load_{name}()")
    current.metadata["data_label"] = data.DESCR
    _df_x = pd.DataFrame(data.data)
    for i, name in enumerate(data.feature_names):
        _df_x.rename(columns={i: name}, inplace=True)
    _df_x["target"] = pd.Series(
        pd.Categorical.from_codes(data.target, categories=data.target_names),
        dtype="category",
    )
    current.df = _df_x
    current.metadata["value_labels"] = make_metadata_value_labels(current.df)


def webuse(name, source="stata", preserve_=False, use_local=False, base_path=""):
    if preserve_:
        preserve()

    if source == "stata":
        _webuse_stata(use_local)(name=name)
    elif source == "vega":
        _webuse_vega(use_local)(name=name)
    elif source == "seaborn":
        _webuse_seaborn(use_local)(name=name)
    elif source in [
        "rdataset",
        "rdatasets",
    ]:  # https://www.statsmodels.org/devel/datasets/index.html#using-datasets-from-r
        _webuse_rdataset(use_local)(name=name)
    elif (
        source == "statsmodels"
    ):  # https://www.statsmodels.org/devel/datasets/index.html#available-datasets
        _webuse_statsmodels(use_local)(name=name)
    elif source in ["datasets", "huggingface"]:  # hugging face
        _webuse_huggingface(use_local)(name=name)
    elif source == "sklearn":
        _webuse_sklearn(use_local)(name=name)
    else:
        raise Exception("Source not found.")

    _print("(" + current.metadata["data_label"] + ")")
    _print(current.df)

pdexplorer/webuse.py
pandichef/pdexplorer
from ._dataset import current
from ._print import _print


def xpose() -> None:
    current.df = current.df.transpose()
    _print(current.df)

pdexplorer/xpose.py
pandichef/pdexplorer
"""A setuptools based setup module.
See:
https://packaging.python.org/guides/distributing-packages-using-setuptools/
https://github.com/pypa/sampleproject
Modified by Madoshakalaka@Github (dependency links added)
"""

# Always prefer setuptools over distutils
from setuptools import setup, find_packages
from os import path

# io.open is needed for projects that support Python 2.7
# It ensures open() defaults to text mode with universal newlines,
# and accepts an argument to specify the text encoding
# Python 3 only projects can skip this import
from io import open

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, "README.md"), encoding="utf-8") as f:
    long_description = f.read()

# Arguments marked as "Required" below must be included for upload to PyPI.
# Fields marked as "Optional" may be commented out.

setup(
    # This is the name of your project. The first time you publish this
    # package, this name will be registered for you. It will determine how
    # users can install this project, e.g.:
    #
    # $ pip install sampleproject
    #
    # And where it will live on PyPI: https://pypi.org/project/sampleproject/
    #
    # There are some restrictions on what makes a valid project name
    # specification here:
    # https://packaging.python.org/specifications/core-metadata/#name
    name="pdexplorer",  # Required
    # Versions should comply with PEP 440:
    # https://www.python.org/dev/peps/pep-0440/
    #
    # For a discussion on single-sourcing the version across setup.py and the
    # project code, see
    # https://packaging.python.org/en/latest/single_source_version.html
    version="0.0.31",  # Required
    # This is a one-line description or tagline of what your project does. This
    # corresponds to the "Summary" metadata field:
    # https://packaging.python.org/specifications/core-metadata/#summary
    description="A Stata emulator for Python/pandas",  # Optional
    # This is an optional longer description of your project that represents
    # the body of text which users will see when they visit PyPI.
    #
    # Often, this is the same as your README, so you can just read it in from
    # that file directly (as we have already done above)
    #
    # This field corresponds to the "Description" metadata field:
    # https://packaging.python.org/specifications/core-metadata/#description-optional
    long_description=long_description,  # Optional
    # Denotes that our long_description is in Markdown; valid values are
    # text/plain, text/x-rst, and text/markdown
    #
    # Optional if long_description is written in reStructuredText (rst) but
    # required for plain-text or Markdown; if unspecified, "applications should
    # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and
    # fall back to text/plain if it is not valid rst" (see link below)
    #
    # This field corresponds to the "Description-Content-Type" metadata field:
    # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional
    long_description_content_type="text/markdown",  # Optional (see note above)
    # This should be a valid link to your project's main homepage.
    #
    # This field corresponds to the "Home-Page" metadata field:
    # https://packaging.python.org/specifications/core-metadata/#home-page-optional
    # url="https://github.com/pypa/sampleproject",  # Optional
    # This should be your name or the name of the organization which owns the
    # project.
    author="pandichef",  # Optional
    # This should be a valid email address corresponding to the author listed
    # above.
    # author_email="pypa-dev@googlegroups.com",  # Optional
    # Classifiers help users find your project by categorizing it.
    #
    # For a list of valid classifiers, see https://pypi.org/classifiers/
    classifiers=[  # Optional
        # How mature is this project? Common values are
        #   3 - Alpha
        #   4 - Beta
        #   5 - Production/Stable
        "Development Status :: 3 - Alpha",
        # Indicate who your project is intended for
        "Intended Audience :: Developers",
        "Topic :: Software Development :: Build Tools",
        # Pick your license as you wish
        "License :: OSI Approved :: BSD License",
        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        # These classifiers are *not* checked by 'pip install'. See instead
        # 'python_requires' below.
        # "Programming Language :: Python :: 3.6",
        # "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
        "Programming Language :: Python :: 3.11",
    ],
    # This field adds keywords for your project which will appear on the
    # project page. What does your project relate to?
    #
    # Note that this is a string of words separated by whitespace, not a list.
    # keywords="sample setuptools development",  # Optional
    # You can just specify package directories manually here if your project is
    # simple. Or you can use find_packages().
    #
    # Alternatively, if you just want to distribute a single Python file, use
    # the `py_modules` argument instead as follows, which will expect a file
    # called `my_module.py` to exist:
    #
    #   py_modules=["my_module"],
    #
    packages=find_packages(exclude=["contrib", "docs", "tests"]),  # Required
    # Specify which Python versions you support. In contrast to the
    # 'Programming Language' classifiers above, 'pip install' will check this
    # and refuse to install the project if the version does not match. If you
    # do not support Python 2, you can simplify this to '>=3.5' or similar, see
    # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires
    python_requires=">=3.8, <4",
    # This field lists other packages that your project depends on to run.
    # Any package you put here will be installed by pip when your project is
    # installed, so they must be valid existing projects.
    #
    # For an analysis of "install_requires" vs pip's requirements files see:
    # https://packaging.python.org/en/latest/requirements.html
    install_requires=[
        "requests",
        # "pandas>=1.0.0",  # DataFrame.to_markdown added in version 1.0.0
        "pandas",
        "scipy",  # see mtgyield()
        # "seaborn",  # Use Altair instead
        "altair",  # for all graphing (note that matplotlib, seaborn, bokeh, etc are not used)
        "vegafusion[embed]",  # provides the alt.Chart.transformed_data method
        "vega_datasets",  # data from Vega (which Altair is based on)
        "statsmodels",  # Econometrics
        "scikit-learn",  # Machine Learning (excluding neural networks)
        "graphviz",  # to visualize decision trees
        "torch",  # PyTorch for Neural networks
        "huggingface_hub",  # for logging into huggingface
        "transformers[torch]",  # tests fail without [torch]
        "evaluate",  # hf tests failed to pass without this
        # "soundfile",  # huggingface audio classification
        # "librosa",  # huggingface audio classification
        "openai",  # for ftgpt
        # "accelerate"  # hf tests failed to pass without this
        "datasets",  # data from Hugging Face (associated with PyTorch)
        "ydata-profiling",  # for cf command
        "sweetviz",  # for cf command
        "dtale",  # for browse command
        "gql[all]",  # for thegraph
        "selenium-wire",  # for thegraph
        # "keyboard",
        "pynput",  # faster alternative to keyboard
        "selenium",  # since webbrowser.open is broken
        # "streamlit",
        # "bardapi",
        # "openai",  # requires API Key
        # "pandas_market_calendars",  # dates for financial markets
        # "numpy-financial",  # basic financial functions
    ],  # Optional
    # List additional groups of dependencies here (e.g. development
    # dependencies). Users will be able to install these using the "extras"
    # syntax, for example:
    #
    #   $ pip install sampleproject[dev]
    #
    # Similar to `install_requires` above, these must be valid existing
    # projects.
    extras_require={"dev": ["ipython", "pytest",]},  # Optional
    # If there are data files included in your packages that need to be
    # installed, specify them here.
    #
    # Sometimes you’ll want to use packages that are properly arranged with
    # setuptools, but are not published to PyPI. In those cases, you can specify
    # a list of one or more dependency_links URLs where the package can
    # be downloaded, along with some additional hints, and setuptools
    # will find and install the package correctly.
    # see https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi
    #
    dependency_links=[],
    # If using Python 2.6 or earlier, then these have to be included in
    # MANIFEST.in as well.
    # package_data={"sample": ["package_data.dat"]},  # Optional
    # Although 'package_data' is the preferred approach, in some case you may
    # need to place data files outside of your packages. See:
    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files
    #
    # In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
    # data_files=[("my_data", ["data/data_file"])],  # Optional
    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # `pip` to create the appropriate form of executable for the target
    # platform.
    #
    # For example, the following would provide a command called `sample` which
    # executes the function `main` from this package when invoked:
    # entry_points={"console_scripts": ["sample=sample:main"]},  # Optional
    # List additional URLs that are relevant to your project as a dict.
    #
    # This field corresponds to the "Project-URL" metadata fields:
    # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use
    #
    # Examples listed include a pattern for specifying where the package tracks
    # issues, where the source is hosted, where to say thanks to the package
    # maintainers, and where to support the project financially. The key is
    # what's used to render the link text on PyPI.
    project_urls={  # Optional
        "Bug Reports": "https://github.com/pandichef/pdexplorer/issues",
        # "Funding": "https://donate.pypi.org",
        # "Say Thanks!": "http://saythanks.io/to/example",
        "Source": "https://github.com/pandichef/pdexplorer",
    },
)

setup.py
pandichef/pdexplorer