MLup framework for easy and fast run ML in production
import mlup
from mlup import constants


if __name__ == "__main__":
    up = mlup.UP(
        conf=mlup.Config(
            name='MyName',
            version='MyVersion',
            columns=[
                {'name': 'MinTemp', 'type': 'float', 'required': False, 'default': 1.4},
                {'name': 'MaxTemp', 'type': 'float'},
                {'name': 'Humidity9am', 'type': 'float'},
                {'name': 'Humidity3pm', 'type': 'float'},
                {'name': 'Pressure9am', 'type': 'float'},
                {'name': 'Pressure3pm', 'type': 'float'},
                {'name': 'Temp9am', 'type': 'float'},
                {'name': 'Temp3pm', 'type': 'float'},
            ],
            auto_detect_predict_params=True,
            storage_type=constants.StorageType.disk,
            storage_kwargs={
                'path_to_files': '../mldata/models/scikit-learn-binary_cls_model.pckl',
                'files_mask': 'scikit-learn-binary_cls_model.pckl',
            },
            use_thread_loop=False,
        )
    )
    up.ml.load()
    up.to_yaml('example-config.yaml')
    up.to_json('example-config.json')

    # Load configs from config file
    new_up = mlup.UP.load_from_yaml('example-config.yaml', load_model=True)

    # Save configs to file
    # or up.config_provider.save_to_yaml
    up.to_yaml('example-save-config.yaml')

    # You can load config and then change it
    up_for_change = up.load_from_yaml('example-config.yaml', load_model=True)
    up_for_change.name = 'NewName'
    up_for_change.to_yaml('example-changed-config.yaml')

examples/configs.py
nxexox/pymlup
import mlup
from mlup import constants


up = mlup.UP(
    conf=mlup.Config(
        # Set your config, for work model and get model.
        storage_type=constants.StorageType.disk,
        storage_kwargs={
            'path_to_files': '../mldata/models/scikit-learn-binary_cls_model.pckl',
            'files_mask': 'scikit-learn-binary_cls_model.pckl',
        },
    )
)
up.ml.load()
up.web.load()

# If you want to run the application yourself, or add something else to it, use this variable.
# Example with uvicorn: uvicorn mlupapp:app --host 0.0.0.0 --port 80
app = up.web.app

if __name__ == '__main__':
    # Default port is 8009
    up.run_web_app(daemon=True)

examples/daemon.py
nxexox/pymlup
import mlup


up = mlup.UP.load_from_yaml('example-config.yaml', load_model=True)
# load_model=True will calling up.ml.load()
up.web.load()

# If you want to run the application yourself, or add something else to it, use this variable.
# Example with uvicorn: uvicorn mlupapp:app --host 0.0.0.0 --port 80
app = up.web.app

if __name__ == '__main__':
    up.run_web_app()

examples/from_config.py
nxexox/pymlup
import mlup
from mlup import constants
import uvicorn


up = mlup.UP(
    conf=mlup.Config(
        # Set your config, for work model and get model.
        storage_type=constants.StorageType.disk,
        storage_kwargs={
            'path_to_files': 'mldata/models/scikit-learn-binary_cls_model.pckl',
            'files_mask': 'scikit-learn-binary_cls_model.pckl',
        },
    )
)
up.ml.load()
# For create WebAPp for uvicorn, need call load() - up.web.load().
# load call automate, when you call up.run_web_app().
up.web.load()

# If you want to run the application yourself, or add something else to it, use this variable.
# Example with uvicorn: uvicorn gunicorn_run:app --host 0.0.0.0 --port 80
app = up.web.app

# gunicorn gunicorn_run:app --workers 2 --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:8009
if __name__ == '__main__':
    uvicorn.run("__main__:app", host='0.0.0.0', port=8009, workers=2)

examples/gunicorn_run.py
nxexox/pymlup
from mlup.up import UP, Config, generate_default_config
from mlup.ml.model import MLupModel, ModelConfig
from mlup.web.app import MLupWebApp, WebAppConfig


__version__ = "0.2.2"

mlup/__init__.py
nxexox/pymlup
from dataclasses import dataclass
from typing import Union, Optional

from mlup.constants import ModelLibraryType, ModelDataTransformerType, BinarizationType, WebAppArchitecture
from mlup.up import Config


@dataclass
class WorkerAndQueueConfig(Config):
    mode: WebAppArchitecture = WebAppArchitecture.worker_and_queue


@dataclass
class BatchingConfig(Config):
    mode: WebAppArchitecture = WebAppArchitecture.batching


@dataclass
class ScikitLearnConfig(Config):
    type: Union[str, ModelLibraryType] = ModelLibraryType.SCIKIT_LEARN
    predict_method_name: str = 'predict'


@dataclass
class ScikitLearnWorkerConfig(ScikitLearnConfig):
    mode: WebAppArchitecture = WebAppArchitecture.worker_and_queue


@dataclass
class ScikitLearnBatchingConfig(ScikitLearnConfig):
    mode: WebAppArchitecture = WebAppArchitecture.batching


@dataclass
class TensorflowConfig(Config):
    type: Union[str, ModelLibraryType] = ModelLibraryType.TENSORFLOW
    predict_method_name: str = '__call__'
    data_transformer_for_predict: Union[str, ModelDataTransformerType] = ModelDataTransformerType.TENSORFLOW_TENSOR
    data_transformer_for_predicted: Union[str, ModelDataTransformerType] = ModelDataTransformerType.TENSORFLOW_TENSOR
    dtype_for_predict: Optional[str] = 'float32'


@dataclass
class TensorflowWorkerConfig(TensorflowConfig):
    mode: WebAppArchitecture = WebAppArchitecture.worker_and_queue


@dataclass
class TensorflowBatchingConfig(TensorflowConfig):
    mode: WebAppArchitecture = WebAppArchitecture.batching


@dataclass
class TorchConfig(Config):
    type: Union[str, ModelLibraryType] = ModelLibraryType.TORCH
    predict_method_name: str = '__call__'
    data_transformer_for_predict: Union[str, ModelDataTransformerType] = ModelDataTransformerType.TORCH_TENSOR
    data_transformer_for_predicted: Union[str, ModelDataTransformerType] = ModelDataTransformerType.TORCH_TENSOR
    dtype_for_predict: Optional[str] = 'float32'


@dataclass
class TorchWorkerConfig(TorchConfig):
    mode: WebAppArchitecture = WebAppArchitecture.worker_and_queue


@dataclass
class TorchBatchingConfig(TorchConfig):
    mode: WebAppArchitecture = WebAppArchitecture.batching


@dataclass
class OnnxConfig(Config):
    type: Union[str, ModelLibraryType] = ModelLibraryType.ONNX
    binarization_type: Union[str, BinarizationType] = BinarizationType.ONNX_INFERENCE_SESSION


@dataclass
class OnnxWorkerConfig(OnnxConfig):
    mode: WebAppArchitecture = WebAppArchitecture.worker_and_queue


@dataclass
class OnnxBatchingConfig(OnnxConfig):
    mode: WebAppArchitecture = WebAppArchitecture.batching

mlup/baseline.py
nxexox/pymlup
import inspect
import json
import logging
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import Dict, Any, Set, Union, Optional

import yaml

from mlup.constants import StorageType


def set_logging_settings(logging_config: Dict, level: Optional[int] = None):
    logging.config.dictConfig(logging_config)
    if level:
        logging.basicConfig(level=level)


LOGGING_CONFIG: Dict[str, Any] = {
    "version": 1,
    "disable_existing_loggers": False,
    "formatters": {
        "mlup": {
            "()": "mlup.utils.logging.SwitchFormatter",
            "fmt": "%(levelname)s:[%(asctime)s.%(msecs)03d] - %(message)s",
            "fmt_web": "%(levelprefix)s [%(process)d][%(thread)d] [%(asctime)s.%(msecs)03d] - %(message)s",
            "fmt_console_commands": "%(message)s",
            "datefmt": "%Y-%m-%d %H:%M:%S",
        },
        "uvicorn": {
            "()": "uvicorn.logging.DefaultFormatter",
            "fmt": "%(levelprefix)s %(message)s",
            "use_colors": None,
        },
        "access": {
            "()": "uvicorn.logging.AccessFormatter",
            "fmt": '%(levelprefix)s [%(process)d][%(thread)d] [%(asctime)s.%(msecs)03d] - %(client_addr)s "%(request_line)s" %(status_code)s',  # noqa: E501
        },
    },
    "handlers": {
        "mlup": {
            "formatter": "mlup",
            "class": "logging.StreamHandler",
            "stream": "ext://sys.stdout",
        },
        "uvicorn": {
            "formatter": "uvicorn",
            "class": "logging.StreamHandler",
            "stream": "ext://sys.stderr",
        },
        "access": {
            "formatter": "access",
            "class": "logging.StreamHandler",
            "stream": "ext://sys.stdout",
        },
    },
    "loggers": {
        "mlup": {"handlers": ["mlup"], "propagate": False},
        "uvicorn": {"handlers": ["uvicorn"], "level": "INFO", "propagate": False},
        "uvicorn.error": {"handlers": ["uvicorn"], "level": "INFO", "propagate": False},
        "uvicorn.access": {"handlers": ["access"], "level": "INFO", "propagate": False},
    },
}

set_logging_settings(LOGGING_CONFIG)
logger = logging.getLogger('mlup')
CONFIGS_VERSION = '1'
CONFIGS = {"ml", "web"}
EXCLUDE_KEYS = {
    "ml": {"ml", "_data_transformer_for_predict", "_data_transformer_for_predicted"},
    "web": {"ml", "custom_column_pydantic_model"}
}
EXCLUDE_CONDITION_KEYS = [
    {'condition': {'storage_type': StorageType.memory.value}, 'skip': ['storage_kwargs']}
]


@dataclass
class ConfigProvider:
    obj_for_config: Any
    configs_objects_map: Dict

    def _get_param_names(self, obj: Any) -> Set:
        sign = inspect.signature(obj.__init__)
        return {param_name for param_name in sign.parameters.keys() if param_name.lower() != 'self'}

    def _set_params_to_obj(self, obj, conf: Dict, exclude_keys: Set):
        for k, v in conf.items():
            if k not in exclude_keys:
                setattr(obj.conf, k, v)

    def set_config_from_dict(self, config: Dict):
        for cls_name in CONFIGS:
            obj = getattr(self.obj_for_config, cls_name)
            if obj is not None:
                logger.debug(f'Set params to {obj.__class__}')
                self._set_params_to_obj(obj, config.get(cls_name, {}), EXCLUDE_KEYS.get(cls_name, {}))

    def get_config_dict(self) -> Dict:
        config = {}
        for cls_name in CONFIGS:
            config.setdefault(cls_name, {})
            obj = getattr(self.obj_for_config, cls_name)

            for param_name in self._get_param_names(self.configs_objects_map[cls_name]):
                if param_name in EXCLUDE_KEYS.get(cls_name, {}):
                    continue
                try:
                    val = getattr(obj.conf, param_name)
                except AttributeError as e:
                    raise e
                if isinstance(val, Enum):
                    val = val.value
                config[cls_name][param_name] = val

            for exclude in EXCLUDE_CONDITION_KEYS:
                if all(_k in config[cls_name] and config[cls_name][_k] == _v for _k, _v in exclude['condition'].items()):
                    for skip in exclude['skip']:
                        config[cls_name].pop(skip, None)

        return config

    def load_from_dict(self, conf: Dict):
        logger.info('Load config from dict')
        self.set_config_from_dict(conf)

    def load_from_json(self, file_path: Union[str, Path]):
        logger.info(f'Load config from {file_path}')
        with open(file_path, 'r') as f:
            _conf = json.load(f)
            self.set_config_from_dict(_conf)

    def load_from_yaml(self, file_path: Union[str, Path]):
        logger.info(f'Load config from {file_path}')
        with open(file_path, 'r') as f:
            _conf = yaml.safe_load(f)
            self.set_config_from_dict(_conf)

    def save_to_json(self, file_path: Union[str, Path]):
        logger.info(f'Save config to {file_path}')
        config_dict = self.get_config_dict()
        config_dict['version'] = CONFIGS_VERSION
        with open(file_path, 'w') as f:
            json.dump(config_dict, f)

    def save_to_yaml(self, file_path: Union[str, Path]):
        logger.info(f'Save config to {file_path}')
        config_dict = self.get_config_dict()
        with open(file_path, 'w') as f:
            f.write(f"version: '{CONFIGS_VERSION}'\n")
            yaml.dump(config_dict, f)

mlup/config.py
nxexox/pymlup
import importlib
import sys
from typing import List

import mlup


AVAILABLE_COMMANDS = """Available commands:
  * validate-config     Validate your config for valid format for MLup.
  * run                 Run your mlup application, from different sources.
  * make-app            Create python file with mlup application code by your settings.
"""


HELP = f"""usage: mlup [-h] [-v] command

positional arguments:
  command               Command name for run

options:
  -h, --help            show this help message and exit
  -v, --version         show current version mlup and exit

{AVAILABLE_COMMANDS}
"""


def run_command(args: List[str]):
    try:
        command, _ = args[0], args[1:]
    except IndexError:
        print(HELP)
        sys.exit(1)

    if command.strip() in ('-h', '--help', 'help'):
        print(HELP)
        sys.exit()

    if command.strip() in ('-v', '--version', 'version'):
        print(mlup.__version__)
        sys.exit()

    command = command.replace('-', '_')

    try:
        module = importlib.import_module(f'mlup.console_scripts.{command}')
    except ModuleNotFoundError:
        print(f'Invalid command {command} - mlup.console_scripts.{command}.')
        print(AVAILABLE_COMMANDS)
        sys.exit(1)

    getattr(module, 'main')()


def main():
    args = sys.argv[1:]
    run_command(args)


if __name__ == '__main__':
    main()

mlup/console_scripts/command.py
nxexox/pymlup
import argparse
import logging
import os
import sys
from pathlib import Path
from typing import Optional, Dict

from mlup.console_scripts.utils import get_config_parser, get_set_fields, CHANGE_SETTINGS_HELP_TEXT
from mlup.utils.logging import configure_logging_formatter


configure_logging_formatter('console_commands')
logger = logging.getLogger('mlup')


app_template = """{import_additional_library}


{code_by_load_src}{conf_kwargs}{code_by_load_app}
up.web.load()

# If you want to run the application yourself, or add something else to it, use this variable.
# Example with uvicorn: uvicorn {file_name}:app --host 0.0.0.0 --port 80
app = up.web.app

if __name__ == '__main__':
    up.run_web_app()
"""

free_app_load_template = """# You can load the model yourself and pass it to the "ml_model" argument.
# up = mlup.UP(ml_model=my_model, conf=mlup.Config())
up = mlup.UP(
    conf=mlup.Config(
        # Set your config, for work model and get model.
        # You can use storage_type and storage_kwargs for auto_load model from storage.{conf_kwargs}
    )
)
"""

from_model_app_load_template = """up = mlup.UP(
    conf=mlup.Config(
        # Set your config, for work model and get model.{conf_kwargs}
    )
)
"""

pickle_load_template = """with open('{path_to_bin_file}', 'rb') as f:
    up = pickle.load(f)
"""


class MakeAppError(Exception):
    pass


class InputArgsError(MakeAppError):
    pass


class GenerateCodeError(MakeAppError):
    pass


class SaveAppError(MakeAppError):
    pass


def _conf_kwargs_to_str(
    config_line_prefix: str = '',
    kv_sep: str = '=',
    end_sep: str = ',',
    **config_kwargs
) -> str:
    kwargs_str = ''
    for k, v in config_kwargs.items():
        kwargs_str += f"\n{config_line_prefix}{k}{kv_sep}{v}{end_sep}"
    return kwargs_str


def get_app_code_without_preload(path_to_app, **config_kwargs):
    return app_template.format(
        file_name=Path(path_to_app).stem,
        import_additional_library='import mlup',
        code_by_load_src=free_app_load_template.format(
            conf_kwargs=_conf_kwargs_to_str('        ', **config_kwargs),
        ),
        code_by_load_app='up.ml.load()',
        conf_kwargs='',
    )


def get_app_code_from_conf(path_to_app: str, config_path: str, config_type: str, **config_kwargs) -> str:
    if config_type == 'json':
        return app_template.format(
            file_name=Path(path_to_app).stem,
            import_additional_library='import mlup',
            code_by_load_src=f"up = mlup.UP.load_from_json('{config_path}', load_model=False)",
            conf_kwargs=_conf_kwargs_to_str('up.conf.', kv_sep=' = ', end_sep='', **config_kwargs),
            code_by_load_app='\nup.ml.load()',
        )
    elif config_type == 'yaml':
        return app_template.format(
            file_name=Path(path_to_app).stem,
            import_additional_library='import mlup',
            code_by_load_src=f"up = mlup.UP.load_from_yaml('{config_path}', load_model=False)",
            conf_kwargs=_conf_kwargs_to_str('up.conf.', kv_sep=' = ', end_sep='', **config_kwargs),
            code_by_load_app='\nup.ml.load()',
        )
    raise GenerateCodeError(f'Config type {config_type} not supported.')


def get_app_code_from_up_bin(path_to_app: str, binary_path: str, binary_type: str, **config_kwargs) -> str:
    if binary_type == 'pickle':
        return app_template.format(
            file_name=Path(path_to_app).stem,
            import_additional_library='import pickle',
            code_by_load_src=pickle_load_template.format(path_to_bin_file=binary_path),
            conf_kwargs=_conf_kwargs_to_str('up.conf.', kv_sep=' = ', end_sep='', **config_kwargs),
            code_by_load_app='\nif not up.ml.loaded:\n    up.ml.load()',
        )
    elif binary_type == 'joblib':
        return app_template.format(
            file_name=Path(path_to_app).stem,
            import_additional_library='import joblib',
            code_by_load_src=f"up = joblib.load('{binary_path}')\n",
            conf_kwargs=_conf_kwargs_to_str('up.conf.', kv_sep=' = ', end_sep='', **config_kwargs),
            code_by_load_app='\nif not up.ml.loaded:\n    up.ml.load()',
        )
    raise GenerateCodeError(f'Binary type {binary_type} not supported.')


def get_app_code_from_model_bin(path_to_app: str, model_path: str, **model_config_kwargs):
    cp = '        '  # config line prefix
    kwargs_str = _conf_kwargs_to_str('        ', **model_config_kwargs)
    return app_template.format(
        file_name=Path(path_to_app).stem,
        import_additional_library='import mlup\nfrom mlup import constants',
        code_by_load_src=from_model_app_load_template.format(
            conf_kwargs=f"\n{cp}storage_type=constants.StorageType.disk,"
                        f"\n{cp}storage_kwargs=" + "{"
                        f"\n{cp}    'path_to_files': '{model_path}',"
                        f"\n{cp}    'files_mask': '{Path(model_path).name}',"
                        f"\n{cp}" + '},'
                        f"{kwargs_str}"
        ),
        code_by_load_app='up.ml.load()',
        conf_kwargs='',
    )


def save_app(path_to_app: str, app_code: str, force: bool = False):
    if not force:
        if os.path.exists(path_to_app):
            raise SaveAppError(f'File "{path_to_app}" exists. For replace, use argument force=True')

    folder = os.path.dirname(path_to_app)
    if folder:
        os.makedirs(folder, exist_ok=True)

    with open(path_to_app, 'w') as f:
        f.write(app_code)


def make_app(
    path_to_file: str,
    path_to_src: Optional[str] = None,
    use_config: bool = False,
    use_binary: bool = False,
    use_model: bool = False,
    config_type: str = 'yaml',
    binary_type: str = 'pickle',
    force: bool = False,
    extension: str = '.py',
    config_fields: Optional[Dict] = None,
):
    if any(((use_config and use_binary), (use_config and use_model), (use_binary and use_model))):
        raise InputArgsError(
            'You can specify only one of the arguments: use_config [-c], use_binary [-b], use_model [-m].'
        )

    if (use_config or use_binary or use_model) and not path_to_src:
        raise InputArgsError(
            'For arguments use_config [-c], use_binary [-b], use_model [-m] need set argument path_to_src [-s].'
        )

    if path_to_src and not os.path.exists(path_to_src):
        raise FileNotFoundError(f'File "{path_to_src}" not exists.')

    if os.path.isfile(path_to_file) and not force:
        raise FileExistsError(f'File "{path_to_file}" exists.')

    if not path_to_file.endswith(extension):
        path_to_file += extension

    if use_config:
        code = get_app_code_from_conf(
            path_to_app=path_to_file,
            config_path=path_to_src,
            config_type=config_type,
            **(config_fields or {})
        )
    elif use_binary:
        code = get_app_code_from_up_bin(
            path_to_app=path_to_file,
            binary_path=path_to_src,
            binary_type=binary_type,
            **(config_fields or {})
        )
    elif use_model:
        code = get_app_code_from_model_bin(
            path_to_app=path_to_file,
            model_path=path_to_src,
            **(config_fields or {})
        )
    else:
        code = get_app_code_without_preload(path_to_app=path_to_file)

    try:
        save_app(path_to_app=path_to_file, app_code=code, force=force)
        logger.info(f'App success created: {os.path.abspath(path_to_file)}')
    except SaveAppError:
        logger.error(f'App not created: {path_to_file}')
        raise


def main():
    parser = argparse.ArgumentParser(
        'mlup make-app',
        usage='%(prog)s path_to_app\nFor more information use "%(prog)s --help"',
        description="Command for make .py file with web_app and your conf or model.\n"
                    "You can specify one of the arguments:\n"
                    "  -m - for make app with load from binary model file.\n"
                    "  -c - for make app with load from conf file.\n"
                    "  -b - for make app with load from mlup.UP binary object.\n"
                    "OR don't say anything, and than app will be with empty settings.",
        epilog=f'{CHANGE_SETTINGS_HELP_TEXT}\n'
               'After generate python web app file, you can run with python command or uvicorn command:\n'
               '    python /path/to/generated/web_app.py\n'
               '    cd /path/to/generated && uvicorn your_file_name:app --host 0.0.0.0 --port 8009\n'
               '\nExamples:\n'
               '  Make from model:\n'
               '    mlup make-app -ms ~/my_model.onnx ~/web_app.py -f  # With default settings\n'
               '    mlup make-app -ms ~/my_model.onnx ~/web_app.py --up.port=8001 '
               '--up.predict_method_name=\\"__call__\\" --up.use_thread_loop=True '
               '--up.columns=\'[{"name": "col", "type": "list"}]\'  # With custom settings\n'
               '\n  Make from config:\n'
               '    mlup make-app -cs ~/mlup-conf.yaml ~/web_app.py\n'
               '    mlup make-app -cs ~/mlup-conf.json -ct json ~/web_app.py\n'
               '    mlup make-app -cs ~/mlup-conf.yaml ~/web_app.py --up.port=8001 '
               '--up.predict_method_name=\\"__call__\\" --up.use_thread_loop=True '
               '--up.columns=\'[{"name": "col", "type": "list"}]\'  # Add custom settings\n'
               '\n  Make from binary mlup.UP object:\n'
               '    mlup make-app -bs ~/mlup.pkl ~/web_app.py\n'
               '    mlup make-app -bs ~/mlup.joblib -bt joblib ~/web_app.py'
               '    mlup make-app -bs ~/mlup.pkl ~/web_app.py --up.port=8001 '
               '--up.predict_method_name=\\"__call__\\" --up.use_thread_loop=True '
               '--up.columns=\'[{"name": "col", "type": "list"}]\'  # Add custom settings',
        formatter_class=argparse.RawTextHelpFormatter,
    )
    # Add hide argument for command name, because command run from two commands: mlup <command_name>.
    parser.add_argument('command_name', type=str, help=argparse.SUPPRESS)
    parser.add_argument('path_to_app', type=str, help='Path to result file with web app.')
    parser.add_argument(
        '-m', '--model', action='store_true',
        help='Make app from binary file with only model.'
    )
    parser.add_argument(
        '-c', '--conf', action='store_true',
        help='Make app from config file. Configuration file type can be specified with --conf-type.'
    )
    parser.add_argument(
        '-b', '--bin', action='store_true',
        help='Make app from binary file with mlup.UP object and model. '
             'Binarization type can be specified with argument --bin-type.'
    )
    parser.add_argument(
        '-s', '--src', type=str,
        help='Path to config or binary file with mlup.UP object or model. Use for make preload code.'
    )
    parser.add_argument('-f', '--force', action='store_true', help='Replace file by path, if --src file exists.')
    parser.add_argument(
        '-ct', '--conf-type', nargs='?', default='yaml', const='yaml', choices=['json', 'yaml'],
        help='Type config file. Can use only: yaml, json. Default is yaml.'
    )
    parser.add_argument(
        '-bt', '--bin-type', nargs='?', default='pickle', const='pickle', choices=['pickle', 'joblib'],
        help='Type binarization your mlup.UP. Can use only: pickle, joblib. Default is pickle.'
    )

    # Add mlup Config params, for run from model [-m]
    patched_parser = get_config_parser(parser)
    args, unknown = patched_parser.parse_known_args()

    try:
        make_app(
            path_to_file=args.path_to_app,
            path_to_src=args.src,
            use_config=args.conf,
            use_binary=args.bin,
            use_model=args.model,
            config_type=args.conf_type,
            binary_type=args.bin_type,
            force=args.force,
            config_fields=get_set_fields(patched_parser)
        )
    except (InputArgsError, FileExistsError, FileNotFoundError) as e:
        logger.error(str(e))
        sys.exit(1)
    except MakeAppError as e:
        logger.error(str(e))
        sys.exit(1)
    except Exception as e:
        logger.exception(f'Error with create app: "{e}"')
        sys.exit(1)

mlup/console_scripts/make_app.py
nxexox/pymlup
import argparse
import logging
import os
import pickle
import sys
from typing import Dict

import mlup
from mlup.console_scripts.utils import get_config_parser, get_set_fields, CHANGE_SETTINGS_HELP_TEXT
from mlup.constants import StorageType


logger = logging.getLogger('mlup')


class InputArgsError(Exception):
    pass


def run_from_config(config_path: str, config_type: str):
    if config_type == 'json':
        up = mlup.UP.load_from_json(conf_path=config_path, load_model=True)
        up.run_web_app()
    elif config_type == 'yaml':
        up = mlup.UP.load_from_yaml(conf_path=config_path, load_model=True)
        up.run_web_app()
    else:
        raise ValueError(f'Config type {config_type} not supported.')


def run_from_up_bin(binary_path: str, binary_type: str):
    if binary_type == 'pickle':
        with open(binary_path, 'rb') as f:
            up = pickle.load(f)
        up.run_web_app()
    elif binary_type == 'joblib':
        import joblib
        up = joblib.load(binary_path)
        up.run_web_app()
    else:
        raise ValueError(f'Binary type {binary_type} not supported.')


def run_from_model(model_path: str, **kwargs):
    if 'binarization_type' not in kwargs:
        kwargs['binarization_type'] = 'auto'
    up = mlup.UP(
        conf=mlup.Config(
            storage_type=StorageType.disk,
            storage_kwargs={
                'path_to_files': model_path,
                'files_mask': r'.+',
            },
            **kwargs,
        )
    )
    up.ml.load()
    up.run_web_app()


def run(
    path: str,
    conf_type: str = 'yaml',
    bin_type: str = 'pickle',
    use_model: bool = False,
    use_conf: bool = False,
    use_bin: bool = False,
    config_fields: Dict = None,
    verbose: bool = False,
):
    if verbose:
        logging.basicConfig()
        logging.getLogger('mlup').setLevel(logging.DEBUG)
        # logger.setLevel(logging.DEBUG)
        logger.debug('Run in verbose mode')

    if any(((use_conf and use_bin), (use_conf and use_model), (use_bin and use_model))):
        raise InputArgsError('You can specify only one of the arguments: use_conf [-c], use_bin [-b], use_model [-m].')
    if not use_conf and not use_bin and not use_model:
        raise InputArgsError('You must specify one of the arguments: use_conf [-c], use_bin [-b], use_model [-m].')

    if not os.path.exists(path):
        raise FileNotFoundError(f'Path {path} not exists.')

    if use_conf:
        run_from_config(path, conf_type)
    elif use_bin:
        run_from_up_bin(path, bin_type)
    elif use_model:
        run_from_model(path, **config_fields)
    else:
        raise ValueError(
            f"Something went wrong.\n"
            f"path='{path}', conf_type='{conf_type}', bin_type='{bin_type}', use_conf={use_conf}, use_bin={use_bin}."
        )


def main():
    parser = argparse.ArgumentParser(
        'mlup run',
        usage='%(prog)s path\nFor more information use "%(prog)s --help"',
        description="Command for run web application with uvicorn python interface.\n"
                    "You must specify one of the arguments:\n"
                    "  -m - for run web app from binary model file.\n"
                    "  -c - for run web app from config file.\n"
                    "  -b - for run web app with load from mlup.UP binary object.\n\n"
                    "You can set custom uvicorn settings in conf.uvicorn_kwargs dictionary or --up.uvicorn_kwargs. "
                    "See examples.",
        epilog=f'{CHANGE_SETTINGS_HELP_TEXT}\n'
               'Examples:\n'
               '  Run from model:\n'
               '    mlup run -m ~/my_model.onnx  # With default settings\n'
               '    mlup run -m ~/mu_model.onnx --up.port=8011 '
               '--up.binarization_type=\\"mlup.ml.binarization.onnx.InferenceSessionBinarizer\\" '
               '--up.uvicorn_kwargs=\'{"workers": 4, "timeout_graceful_shutdown": 10}\' '
               '--up.use_thread_loop=True --up.columns=\'[{"name": "col", "type": "list"}]\'  # Add custom settings\n'
               '\n  Run from config:\n'
               '    mlup run -c ~/mlup-conf.yaml\n'
               '    mlup run -c ~/mlup-conf.json -ct json\n'
               '    mlup run -c ~/mlup-conf.yaml '
               '--up.uvicorn_kwargs=\'{"workers": 4, "timeout_graceful_shutdown": 10}\'  # Add custom settings\n'
               '\n  Run from binary mlup.UP object:\n'
               '    mlup run -b ~/mlup.pkl\n'
               '    mlup run -b ~/mlup.joblib -bt joblib\n'
               '    mlpu run -b ~/mlup.pkl '
               '--up.uvicorn_kwargs=\'{"workers": 4, "timeout_graceful_shutdown": 10}\'  # Add custom settings',
        formatter_class=argparse.RawTextHelpFormatter,
    )
    # Add hide argument for command name, because command run from two commands: mlup <command_name>.
    parser.add_argument('command_name', type=str, help=argparse.SUPPRESS)
    parser.add_argument('path', type=str, help='Path to config or binary file with mlup.UP object or model.')
    parser.add_argument('-m', '--model', action='store_true', help='Run web app from binary model file.')
    parser.add_argument(
        '-c', '--conf', action='store_true',
        help='Run web app from config file. Configuration file type can be specified with argument --conf-type.'
    )
    parser.add_argument(
        '-b', '--bin', action='store_true',
        help='Run web app from binary file with mlup.UP object. '
             'Binarization type can be specified with argument --bin-type.'
    )
    parser.add_argument(
        '-ct', '--conf-type', nargs='?', default='yaml', const='yaml', choices=['json', 'yaml'],
        help='Type config file. Can use only: yaml, json. Default is yaml.'
    )
    parser.add_argument(
        '-bt', '--bin-type', nargs='?', default='pickle', const='pickle', choices=['pickle', 'joblib'],
        help='Type binarization your mlup.UP object. Can use only: pickle, joblib. Default is pickle.'
    )
    parser.add_argument(
        '-v', '--verbose', action='store_true',
        help='Set DEBUG logger level in application, for see more information about work application.\n'
             'Default logger level is INFO.'
    )

    # Add mlup Config params, for run from model [-m]
    patched_parser = get_config_parser(parser)
    args, unknown = patched_parser.parse_known_args()

    try:
        run(
            path=args.path,
            conf_type=args.conf_type,
            bin_type=args.bin_type,
            use_conf=args.conf,
            use_bin=args.bin,
            use_model=args.model,
            config_fields=get_set_fields(patched_parser),
            verbose=args.verbose,
        )
    except InputArgsError as e:
        parser.error(str(e))
        sys.exit(1)
    except FileNotFoundError as e:
        logger.error(str(e))
        sys.exit(1)

mlup/console_scripts/run.py
nxexox/pymlup
import argparse
from dataclasses import asdict
from typing import Dict, Any

import mlup


CHANGE_SETTINGS_HELP_TEXT = (
    'You can change any setting that is available in mlup.Config by specifying '
    '--up.{conf_variable_name}={your_value} of the setting.\n'
    'Examples for different data types:\n'
    '  --up.port=8011\n'
    '  --up.batch_worker_timeout=10.0\n'
    '  --up.predict_method_name=\\"__call__\\"\n'
    '  --up.use_thread_loop=False\n'
    '  --up.columns=\'[{"name": "col", "type": "list"}]\'\n'
    '  --up.uvicorn_kwargs=\'{"workers": 4, "timeout_graceful_shutdown": 10}\'\n'
)


def get_config_parser(parent_parser: argparse.ArgumentParser):
    parser = argparse.ArgumentParser(
        parent_parser.prog,
        usage=parent_parser.usage,
        description=parent_parser.description,
        epilog=parent_parser.epilog,
        formatter_class=parent_parser.formatter_class,
        parents=[parent_parser],
        add_help=False
    )

    for _f_name in asdict(mlup.Config()):
        if _f_name.startswith('_'):
            continue
        parser.add_argument('--up.' + _f_name, type=str, help=argparse.SUPPRESS, default=argparse.SUPPRESS)

    return parser


def get_set_fields(parser: argparse.ArgumentParser) -> Dict[str, Any]:
    args, _ = parser.parse_known_args()
    result = {}
    for arg_name, arg_value in args.__dict__.items():
        if not arg_name.startswith('up.'):
            continue
        arg_name = arg_name[3:]
        result[arg_name] = arg_value
    return result

mlup/console_scripts/utils.py
nxexox/pymlup
import argparse
import logging
import os
import sys

import mlup
from mlup.utils.logging import configure_logging_formatter


configure_logging_formatter('console_commands')
logger = logging.getLogger('mlup')


class ValidationConfigError(Exception):
    pass


def validate_config(config_path: str, config_type: str):
    if not os.path.exists(config_path):
        raise FileNotFoundError(f'File "{config_path}" not found.')

    if config_type == 'json':
        mlup.UP.load_from_json(conf_path=config_path, load_model=False)
    elif config_type == 'yaml':
        mlup.UP.load_from_yaml(conf_path=config_path, load_model=False)
    else:
        raise ValidationConfigError(f'Config type {config_type} not supported.')


def main():
    parser = argparse.ArgumentParser(
        'mlup validate-config',
        usage='%(prog)s path\nFor more information use "%(prog)s --help"',
        description="Command for validate your config file on semantic correct.\n"
                    "This only runs the code depending on your arguments:\n"
                    "  --type=json - mlup.UP.load_from_json(json_conf_path, load_model=False)\n"
                    "  --type=yaml - mlup.UP.load_from_yaml(yaml_conf_path, load_model=False)",
        epilog="Examples:\n"
               "  mlup validate-config /path/to/my/config/file.yaml\n"
               "  mlup validate-config --type=json /path/to/my/config/file.json",
        formatter_class=argparse.RawTextHelpFormatter,
    )
    # Add hide argument for command name, because command run from two commands: mlup <command_name>.
    parser.add_argument('command_name', type=str, help=argparse.SUPPRESS)
    parser.add_argument('path', type=str, help='Path to file with config.')
    parser.add_argument(
        '--type', nargs='?', default='yaml', const='yaml', choices=['json', 'yaml'],
        help='Type config file. Can use only: yaml, json. Default is yaml.'
    )

    args, unknown = parser.parse_known_args()

    try:
        validate_config(args.path, args.type)
        logger.info('Config is valid')
    except ValidationConfigError as e:
        logger.error(str(e))
        sys.exit(1)
    except FileNotFoundError as e:
        logger.error(str(e))
        sys.exit(1)
    except Exception as e:
        logger.exception(f'Config not is valid: "{e}"')
        sys.exit(1)

mlup/console_scripts/validate_config.py
nxexox/pymlup
from enum import Enum
from io import BufferedIOBase
from typing import NamedTuple, Union, Callable, Optional

IS_X = 'is_X'
THERE_IS_ARGS = 'there_is_args'
DEFAULT_X_ARG_NAME = 'data_for_predict'
ITEM_ID_COL_NAME: str = 'mlup_item_id'
PREDICT_ID_HEADER = 'X-Predict-id'


class LoadedFile(NamedTuple):
    raw_data: Union[str, bytes, BufferedIOBase, Callable] = b''
    path: Optional[str] = None


class ModelLibraryType(Enum):
    SKLEARN = 'sklearn'
    SCIKIT_LEARN = 'scikit-learn'
    LIGHTGMB = 'lightgbm'
    TENSORFLOW = 'tensorflow'
    TORCH = 'torch'
    ONNX = 'onnx_inference_session'
    other = 'other'


class StorageType(Enum):
    memory = 'mlup.ml.storage.memory.MemoryStorage'
    disk = 'mlup.ml.storage.local_disk.DiskStorage'


class BinarizationType(Enum):
    MEMORY = 'mlup.ml.binarization.memory.MemoryBinarizer'
    JOBLIB = 'mlup.ml.binarization.joblib.JoblibBinarizer'
    PICKLE = 'mlup.ml.binarization.pickle.PickleBinarizer'
    LIGHTGBM = 'mlup.ml.binarization.lightgbm.LightGBMBinarizer'
    TENSORFLOW = 'mlup.ml.binarization.tensorflow.TensorFlowBinarizer'
    TENSORFLOW_ZIP = 'mlup.ml.binarization.tensorflow.TensorFlowSavedBinarizer'
    TORCH = 'mlup.ml.binarization.torch.TorchBinarizer'
    TORCH_JIT = 'mlup.ml.binarization.torch.TorchJITBinarizer'
    ONNX_INFERENCE_SESSION = 'mlup.ml.binarization.onnx.InferenceSessionBinarizer'
    ONNX_INFERENCE_SESSION_FULL_RETURN = 'mlup.ml.binarization.onnx.InferenceSessionFullReturnBinarizer'


class ModelDataTransformerType(Enum):
    PANDAS_DF = 'mlup.ml.data_transformers.pandas_data_transformer.PandasDataFrameTransformer'
    NUMPY_ARR = 'mlup.ml.data_transformers.numpy_data_transformer.NumpyDataTransformer'
    TENSORFLOW_TENSOR = 'mlup.ml.data_transformers.tf_tensor_data_transformer.TFTensorDataTransformer'
    TORCH_TENSOR = 'mlup.ml.data_transformers.torch_tensor_data_transformer.TorchTensorDataTransformer'
    SRC_TYPES = 'mlup.ml.data_transformers.src_data_transformer.SrcDataTransformer'


class WebAppArchitecture(Enum):
    directly_to_predict = 'mlup.web.architecture.directly_to_predict.DirectlyToPredictArchitecture'
    worker_and_queue = 'mlup.web.architecture.worker_and_queue.WorkerAndQueueArchitecture'
    batching = 'mlup.web.architecture.batching.BatchingSingleProcessArchitecture'

mlup/constants.py
nxexox/pymlup
from dataclasses import dataclass


class MLupError(Exception):
    pass


class LoadError(MLupError):
    pass


class ModelLoadError(LoadError):
    pass


class WebAppLoadError(LoadError):
    pass


class ModelBinarizationError(ModelLoadError):
    pass


class PredictError(MLupError):
    http_status: int = 500
    type: str = 'predict_error'


@dataclass
class PredictWaitResultError(PredictError):
    msg: str
    predict_id: str
    http_status: int = 408
    type: str = 'predict_wait_result_error'

    def __str__(self):
        return self.msg


class PredictTransformDataError(PredictError):
    type: str = 'predict_transform_data_error'


@dataclass
class PredictValidationInnerDataError(PredictError):
    msg: str
    predict_id: str
    http_status: int = 422
    type: str = 'validation_error'

    def __str__(self):
        return self.msg

mlup/errors.py
nxexox/pymlup
import abc
from dataclasses import dataclass

from mlup.constants import LoadedFile


@dataclass
class BaseBinarizer(metaclass=abc.ABCMeta):
    @classmethod
    @abc.abstractmethod
    def deserialize(cls, data: LoadedFile):
        pass

    @classmethod
    def is_this_type(cls, loaded_file: LoadedFile) -> float:
        return 0.0

mlup/ml/binarization/base.py
nxexox/pymlup
import logging
from io import BytesIO

from mlup.ml.binarization.pickle import PickleBinarizer
from mlup.constants import LoadedFile
from mlup.utils.profiling import TimeProfiler

try:
    import joblib
except ImportError:
    joblib = None

from mlup.errors import ModelBinarizationError


logger = logging.getLogger('mlup')


class JoblibBinarizer(PickleBinarizer):
    @classmethod
    def deserialize(cls, data: LoadedFile):
        logger.info('Run deserialization joblib data.')
        if joblib is None:
            logger.error('For use joblib, please install it. pip install joblib.')
            raise ModelBinarizationError('For use joblib, please install it. pip install joblib.')

        with TimeProfiler('Time to deserialization joblib data:'):
            data_reader = data.raw_data

            try:
                if data.raw_data and isinstance(data.raw_data, bytes):
                    data_reader = BytesIO(data.raw_data)
                elif data.path:
                    with open(data.path, 'rb') as f:
                        return joblib.load(f)
                return joblib.load(data_reader)
            except Exception as e:
                raise ModelBinarizationError(f'Error with deserialize model: {e}')

mlup/ml/binarization/joblib.py
nxexox/pymlup
import logging

import lightgbm as lgb

from mlup.errors import ModelBinarizationError
from mlup.ml.binarization.base import BaseBinarizer
from mlup.constants import LoadedFile
from mlup.utils.profiling import TimeProfiler


logger = logging.getLogger('mlup')


class LightGBMBinarizer(BaseBinarizer):
    @classmethod
    def deserialize(cls, data: LoadedFile):
        logger.info('Run deserialization lightgbm data.')
        with TimeProfiler('Time to deserialization lightgbm data:'):
            try:
                if data.path:
                    return lgb.Booster(model_file=data.path)

                if not isinstance(data.raw_data, str):
                    if hasattr(data.raw_data, 'read'):
                        data.raw_data = data.raw_data.read()
                    if isinstance(data.raw_data, bytes):
                        data.raw_data = data.raw_data.decode('utf-8')
                return lgb.Booster(model_str=data.raw_data)
            except Exception as e:
                raise ModelBinarizationError(f'Error with deserialize model: {e}')

    @classmethod
    def _check_type_by_bytes(cls, file_data: str) -> bool:
        # Find version in first rows
        *rows, other = file_data.split('\n', 5)
        return any([r.split('=')[0] == 'version' for r in rows])

    @classmethod
    def is_this_type(cls, loaded_file: LoadedFile) -> float:
        probability = 0.0
        if loaded_file.raw_data:
            file_data = loaded_file.raw_data
            try:
                if isinstance(loaded_file.raw_data, bytes):
                    file_data = loaded_file.raw_data.decode('utf-8')
                if cls._check_type_by_bytes(file_data):
                    probability = 0.8
            except Exception:
                pass
        if loaded_file.path:
            file_data = loaded_file.raw_data
            try:
                with open(loaded_file.path, 'r') as f:
                    file_data = f.read(100)
            except Exception:
                pass
            try:
                if cls._check_type_by_bytes(file_data):
                    probability = 0.8
            except Exception:
                pass
            if str(loaded_file.path).endswith('.txt'):
                probability += 0.05
        return probability

mlup/ml/binarization/lightgbm.py
nxexox/pymlup
import logging

from mlup.ml.binarization.base import BaseBinarizer
from mlup.constants import LoadedFile

logger = logging.getLogger('mlup')


class MemoryBinarizer(BaseBinarizer):
    @classmethod
    def deserialize(cls, data: LoadedFile):
        logger.info('Run deserialization memory data.')
        return data.raw_data

mlup/ml/binarization/memory.py
nxexox/pymlup
import logging
import tempfile
from typing import Union, List

import onnx
import onnxruntime

from mlup.errors import ModelBinarizationError
from mlup.ml.binarization.base import BaseBinarizer
from mlup.constants import LoadedFile
from mlup.utils.profiling import TimeProfiler


logger = logging.getLogger('mlup')


class _InferenceSessionWithPredict(onnxruntime.InferenceSession):
    def format_predict(self, predict_result: List):
        if len(predict_result) > 1:
            return predict_result[:-1]
        return predict_result

    def predict(self, input_data):
        input_name = self.get_inputs()[0].name
        res = self.run(None, {input_name: input_data})
        return self.format_predict(res)


class _InferenceSessionWithPredictFullReturn(_InferenceSessionWithPredict):
    def format_predict(self, predict_result: List):
        return predict_result


class InferenceSessionBinarizer(BaseBinarizer):
    inference_class = _InferenceSessionWithPredict

    @classmethod
    def deserialize(cls, data: LoadedFile):
        logger.info('Run deserialization onnxruntime data.')
        with TimeProfiler('Time to deserialization onnxruntime data:'):
            try:
                _data = data.raw_data
                if data.path:
                    _data = data.path
                elif not isinstance(data.raw_data, str):
                    f = tempfile.NamedTemporaryFile(delete=True, suffix='.keras')
                    if isinstance(data.raw_data, bytes):
                        f.write(data.raw_data)
                    else:
                        f.write(data.raw_data.read())
                    f.seek(0)
                    data.path = f.name
                    _data = f.name
                return cls.inference_class(str(_data))
            except Exception as e:
                raise ModelBinarizationError(f'Error with deserialize model: {e}')

    @classmethod
    def _check_type(cls, model_data: Union[str, bytes]) -> bool:
        try:
            onnx.checker.check_model(model_data)
            return True
        except onnx.checker.ValidationError:
            return False
        except Exception:
            return False

    @classmethod
    def is_this_type(cls, loaded_file: LoadedFile) -> float:
        probability = 0.0
        if LoadedFile.raw_data:
            if cls._check_type(loaded_file.raw_data):
                probability = 0.9
        if loaded_file.path:
            if cls._check_type(loaded_file.path):
                probability = 0.9
            if str(loaded_file.path).endswith('.onnx'):
                probability += 0.05
        return probability


class InferenceSessionFullReturnBinarizer(InferenceSessionBinarizer):
    inference_class = _InferenceSessionWithPredictFullReturn

mlup/ml/binarization/onnx.py
nxexox/pymlup
import logging
import pickle
import pickletools
from io import BufferedIOBase

from mlup.errors import ModelBinarizationError
from mlup.ml.binarization.base import BaseBinarizer
from mlup.constants import LoadedFile
from mlup.utils.profiling import TimeProfiler


logger = logging.getLogger('mlup')


class PickleBinarizer(BaseBinarizer):
    @classmethod
    def deserialize(cls, data: LoadedFile):
        logger.info('Run deserialization pickle data.')
        with TimeProfiler('Time to deserialization pickle data:'):
            try:
                if isinstance(data.raw_data, BufferedIOBase):
                    return pickle.load(data.raw_data)
                elif data.raw_data:
                    return pickle.loads(data.raw_data)
                else:
                    with open(data.path, 'rb') as f:
                        return pickle.load(f)
            except Exception as e:
                raise ModelBinarizationError(f'Error with deserialize model: {e}')

    @classmethod
    def _check_type_by_bytes(cls, file_bytes: bytes) -> bool:
        # Check by first and end bytes
        # https://stackoverflow.com/a/73523239
        start_opcode = pickletools.code2op.get(file_bytes[0:1].decode('latin-1'))
        end_opcode = pickletools.code2op.get(file_bytes[-1:].decode('latin-1'))
        return start_opcode.name == 'PROTO' and end_opcode.name == 'STOP'

    @classmethod
    def is_this_type(cls, loaded_file: LoadedFile) -> float:
        probability = 0.0
        if loaded_file.raw_data:
            file_bytes = loaded_file.raw_data
            try:
                if isinstance(file_bytes, str):
                    file_bytes = file_bytes.encode()
                if cls._check_type_by_bytes(file_bytes):
                    probability = 0.9
            except Exception:
                pass

        if loaded_file.path:
            try:
                with open(loaded_file.path, 'rb') as f:
                    # Read only first and last bytes, for check
                    start = f.read(1)
                    f.seek(-2, 2)
                    end = f.read()
                    if cls._check_type_by_bytes(start + end):
                        probability = 0.9
            except Exception:
                pass
            finally:
                if str(loaded_file.path).endswith('.pckl') or str(loaded_file.path).endswith('.pkl'):
                    probability += 0.05
        return probability

mlup/ml/binarization/pickle.py
nxexox/pymlup
import logging
import os.path
import tempfile

import tensorflow

from mlup.errors import ModelBinarizationError
from mlup.ml.binarization.base import BaseBinarizer
from mlup.constants import LoadedFile
from mlup.utils.profiling import TimeProfiler


logger = logging.getLogger('mlup')


class TensorFlowBinarizer(BaseBinarizer):
    @classmethod
    def deserialize(cls, data: LoadedFile):
        logger.info('Run deserialization tensorflow data.')
        with TimeProfiler('Time to deserialization tensorflow data:'):
            try:
                _data = data.raw_data
                if data.path:
                    _data = data.path
                elif not isinstance(data.raw_data, str):
                    f = tempfile.NamedTemporaryFile(delete=True, suffix='.keras')
                    if isinstance(data.raw_data, bytes):
                        f.write(data.raw_data)
                    else:
                        f.write(data.raw_data.read())
                    f.seek(0)
                    data.path = f.name
                    _data = f.name
                model = tensorflow.keras.models.load_model(str(_data), compile=False)
                model.compile()
                return model
            except Exception as e:
                raise ModelBinarizationError(f'Error with deserialize model: {e}')

    @classmethod
    def is_this_type(cls, loaded_file: LoadedFile) -> float:
        probability = 0.0
        if loaded_file.raw_data:
            try:
                if isinstance(loaded_file.raw_data, str):
                    first_bytes = 'PK\x03'
                else:
                    first_bytes = b'PK\x03'
                if loaded_file.raw_data[:3] == first_bytes:
                    probability = 0.5
            except Exception:
                pass
        if loaded_file.path:
            try:
                with open(loaded_file.path, 'rb') as f:
                    file_data = f.read(10)
                    if file_data[:3] == b'PK\x03':
                        probability = 0.5
            except Exception:
                if loaded_file.path.endswith('.keras') or loaded_file.path.endswith('.h5'):
                    probability += 0.3
        return probability


class TensorFlowSavedBinarizer(BaseBinarizer):
    @classmethod
    def deserialize(cls, data: LoadedFile):
        logger.info('Run deserialization tensorflow SavedModel data.')
        with TimeProfiler('Time to deserialization tensorflow SavedModel data:'):
            try:
                _data = data.raw_data
                if data.path:
                    _data = data.path
                elif not isinstance(data.raw_data, str):
                    f = tempfile.NamedTemporaryFile(delete=True)
                    if isinstance(data.raw_data, bytes):
                        f.write(data.raw_data)
                    else:
                        f.write(data.raw_data.read())
                    f.seek(0)
                    data.path = f.name
                    _data = f.name
                return tensorflow.saved_model.load(str(_data))
            except Exception as e:
                raise ModelBinarizationError(f'Error with deserialize model: {e}')

    @classmethod
    def is_this_type(cls, loaded_file: LoadedFile) -> float:
        probability = 0.0
        if loaded_file.path:
            try:
                if os.path.isdir(loaded_file.path):
                    if any([f.endswith('.pb') for f in os.listdir(loaded_file.path)]):
                        probability = 0.3
            except Exception:
                pass
        return probability

mlup/ml/binarization/tensorflow.py
nxexox/pymlup
import logging
from io import BytesIO, BufferedIOBase

import torch

from mlup.errors import ModelBinarizationError
from mlup.ml.binarization.base import BaseBinarizer
from mlup.constants import LoadedFile
from mlup.utils.profiling import TimeProfiler


logger = logging.getLogger('mlup')


class TorchBinarizer(BaseBinarizer):
    @classmethod
    def deserialize(cls, data: LoadedFile):
        logger.info('Run deserialization torch data.')
        with TimeProfiler('Time to deserialization torch data:'):
            try:
                _data = data.raw_data
                if data.path:
                    _data = open(data.path, 'rb')
                elif not isinstance(data.raw_data, BufferedIOBase):
                    _data = BytesIO(data.raw_data)
                model = torch.load(_data)
                model.eval()
                return model
            except Exception as e:
                raise ModelBinarizationError(f'Error with deserialize model: {e}')
            finally:
                if hasattr(_data, 'close'):
                    _data.close()

    @classmethod
    def is_this_type(cls, loaded_file: LoadedFile) -> float:
        probability = 0.0
        if loaded_file.raw_data:
            try:
                if isinstance(loaded_file.raw_data, str):
                    first_bytes = 'PK\x03'
                else:
                    first_bytes = b'PK\x03'
                if loaded_file.raw_data[:3] == first_bytes:
                    probability = 0.5
            except Exception:
                pass
        if loaded_file.path:
            try:
                with open(loaded_file.path, 'rb') as f:
                    file_data = f.read(10)
                    if file_data[:3] == b'PK\x03':
                        probability = 0.5
            except Exception:
                if isinstance(loaded_file.raw_data, str):
                    if loaded_file.raw_data.endswith('.pth'):
                        probability += 0.3
                elif isinstance(loaded_file.raw_data, bytes):
                    if loaded_file.raw_data.endswith(b'.pth'):
                        probability += 0.3
        return probability


class TorchJITBinarizer(BaseBinarizer):
    @classmethod
    def deserialize(cls, data: LoadedFile):
        logger.info('Run deserialization torch JIT data.')
        with TimeProfiler('Time to deserialization torch JIT data:'):
            try:
                _data = data.raw_data
                if data.path:
                    _data = open(data.path, 'rb')
                elif not isinstance(data.raw_data, BufferedIOBase):
                    _data = BytesIO(data.raw_data)
                model = torch.jit.load(_data)
                model.eval()
                return model
            except Exception as e:
                raise ModelBinarizationError(f'Error with deserialize model: {e}')
            finally:
                if hasattr(_data, 'close'):
                    _data.close()

mlup/ml/binarization/torch.py
nxexox/pymlup
import abc
from dataclasses import dataclass, field
from typing import Any, Dict, Union, List, Optional

from mlup.constants import ModelDataTransformerType


@dataclass
class BaseDataTransformer(metaclass=abc.ABCMeta):
    """Base class for transformers data for predict to model format and back."""
    dtype_name: Optional[str] = None
    dtype: Optional[Any] = None
    DATA_TYPE: ModelDataTransformerType = field(init=False)

    def __post_init__(self):
        if self.dtype_name and self.dtype is None:
            self.dtype = self.get_dtype()

    def get_dtype(self) -> Any:
        return None

    @abc.abstractmethod
    def transform_to_model_format(
        self,
        data: List[Union[Dict[str, Any], List[Any]]],
        columns: Optional[List[Dict[str, str]]] = None,
    ):
        pass

    @abc.abstractmethod
    def transform_to_json_format(self, data: Any):
        pass

mlup/ml/data_transformers/base.py
nxexox/pymlup
import logging
from typing import Any, List, Dict, Union, Optional

import numpy as np

from mlup.constants import ModelDataTransformerType
from mlup.ml.data_transformers.base import BaseDataTransformer


logger = logging.getLogger('mlup')


class NumpyDataTransformer(BaseDataTransformer):
    """Class for transformers data for predict to numpy.ndarray format and back."""
    dtype: Optional[np.generic] = None
    DATA_TYPE: ModelDataTransformerType = ModelDataTransformerType.NUMPY_ARR

    def get_dtype(self) -> np.generic:
        return getattr(np, self.dtype_name)

    def transform_to_model_format(
        self,
        data: List[Union[Dict[str, Any], List[Any]]],
        columns: Optional[List[Dict[str, str]]] = None,
    ) -> np.ndarray:
        result = []
        data_type = None
        columns = columns or []
        for obj in data:
            if not data_type:
                if isinstance(obj, list):
                    data_type = 'list'
                else:
                    data_type = 'dict'

            if data_type == 'list':
                result.append(obj)
            else:
                # Order regularization by columns or columns in first object
                _r = []
                if columns:
                    for c_conf in columns:
                        _r.append(obj[c_conf["name"]])
                else:
                    for _k, _v in obj.items():
                        columns.append({"name": _k})
                        _r.append(_v)
                result.append(_r)

        logger.debug(f'Create numpy.ndarray {len(result)} success.')
        return np.array(result, dtype=self.dtype)

    def transform_to_json_format(self, data: Union[List, np.ndarray]):
        if isinstance(data, List):
            return [d.tolist() for d in data]
        return data.tolist()

mlup/ml/data_transformers/numpy_data_transformer.py
nxexox/pymlup
import logging
from typing import Any, List, Dict, Union, Optional

import pandas
from pandas import DataFrame
from pandas._typing import Dtype

from mlup.constants import ModelDataTransformerType
from mlup.ml.data_transformers.base import BaseDataTransformer


logger = logging.getLogger('mlup')


class PandasDataFrameTransformer(BaseDataTransformer):
    """Class for transform data for predict to DataFrame format and back."""
    dtype: Optional[Dtype] = None
    DATA_TYPE: ModelDataTransformerType = ModelDataTransformerType.PANDAS_DF

    def get_dtype(self) -> Dtype:
        return getattr(pandas, self.dtype_name)()

    def transform_to_model_format(
        self,
        data: List[Union[Dict[str, Any], List[Any]]],
        columns: Optional[List[Dict[str, str]]] = None,
    ) -> DataFrame:
        _data_dict: Dict[str, List] = dict()
        for obj in data:
            if isinstance(obj, list):
                if not columns:
                    raise ValueError('If input data have type list, than need set columns.')
                for column_conf, value in zip(columns, obj):
                    column = column_conf["name"]
                    _data_dict.setdefault(column, [])
                    _data_dict[column].append(value)
            else:
                for column, value in obj.items():
                    _data_dict.setdefault(column, [])
                    _data_dict[column].append(value)

        df = DataFrame(data=_data_dict, dtype=self.dtype)
        logger.debug(f'Create pandas.DataFrame {df.shape} success.')
        return df

    def transform_to_json_format(self, data: Union[List, DataFrame]):
        if isinstance(data, List):
            return [d.to_dict('records') for d in data]
        return data.to_dict('records')

mlup/ml/data_transformers/pandas_data_transformer.py
nxexox/pymlup
from typing import Any, List, Dict, Union, Optional

from mlup.constants import ModelDataTransformerType
from mlup.ml.data_transformers.base import BaseDataTransformer


class SrcDataTransformer(BaseDataTransformer):
    """Class for don't transform data for predict. Use sending client and returning model data."""
    DATA_TYPE: ModelDataTransformerType = ModelDataTransformerType.SRC_TYPES

    def transform_to_model_format(
        self,
        data: List[Union[Dict[str, Any], List[Any]]],
        columns: Optional[List[Dict[str, str]]] = None,
    ) -> List[Union[Dict[str, Any], List[Any]]]:
        return data

    def transform_to_json_format(self, data: Any) -> Any:
        return data

mlup/ml/data_transformers/src_data_transformer.py
nxexox/pymlup
import logging
from typing import Any, List, Dict, Union, Optional

import tensorflow as tf

from mlup.constants import ModelDataTransformerType
from mlup.ml.data_transformers.base import BaseDataTransformer


logger = logging.getLogger('mlup')


class TFTensorDataTransformer(BaseDataTransformer):
    """Class for transformers data for predict to tensorflow.Tensor format and back."""
    dtype: Optional[tf.DType] = None
    DATA_TYPE: ModelDataTransformerType = ModelDataTransformerType.TENSORFLOW_TENSOR

    def get_dtype(self) -> tf.DType:
        return getattr(tf, self.dtype_name)

    def transform_to_model_format(
        self,
        data: List[Union[Dict[str, Any], List[Any]]],
        columns: Optional[List[Dict[str, str]]] = None,
    ) -> tf.Tensor:
        result = []
        data_type = None
        columns = columns or []
        for obj in data:
            if not data_type:
                if isinstance(obj, list):
                    data_type = 'list'
                else:
                    data_type = 'dict'

            if data_type == 'list':
                result.append(obj)
            else:
                # Order regularization by columns or columns in first object
                _r = []
                if columns:
                    for c_conf in columns:
                        _r.append(obj[c_conf["name"]])
                else:
                    for _k, _v in obj.items():
                        columns.append({"name": _k})
                        _r.append(_v)
                result.append(_r)

        logger.debug(f'Create tensorflow.Tensor {len(result)} success.')
        return tf.convert_to_tensor(result, dtype=self.dtype)

    def transform_to_json_format(cls, data: Union[List, tf.Tensor]):
        if isinstance(data, List):
            return [d.numpy().tolist() for d in data]
        return data.numpy().tolist()

mlup/ml/data_transformers/tf_tensor_data_transformer.py
nxexox/pymlup
import logging
from typing import Any, List, Dict, Union, Optional

import torch

from mlup.constants import ModelDataTransformerType
from mlup.ml.data_transformers.base import BaseDataTransformer


logger = logging.getLogger('mlup')


class TorchTensorDataTransformer(BaseDataTransformer):
    """Class for transformers data for predict to pytorch.tensor format and back."""
    dtype: Optional[torch.dtype] = None
    DATA_TYPE: ModelDataTransformerType = ModelDataTransformerType.TORCH_TENSOR

    def get_dtype(self) -> torch.dtype:
        return getattr(torch, self.dtype_name)

    def transform_to_model_format(
        self,
        data: List[Union[Dict[str, Any], List[Any]]],
        columns: Optional[List[Dict[str, str]]] = None,
    ) -> torch.Tensor:
        result = []
        data_type = None
        columns = columns or []
        for obj in data:
            if not data_type:
                if isinstance(obj, list):
                    data_type = 'list'
                else:
                    data_type = 'dict'

            if data_type == 'list':
                result.append(obj)
            else:
                # Order regularization by columns or columns in first object
                _r = []
                if columns:
                    for c_conf in columns:
                        _r.append(obj[c_conf["name"]])
                else:
                    for _k, _v in obj.items():
                        columns.append({"name": _k})
                        _r.append(_v)
                result.append(_r)

        logger.debug(f'Create tensorflow.Tensor {len(result)} success.')
        return torch.tensor(result, dtype=self.dtype)

    def transform_to_json_format(self, data: Union[List, torch.Tensor]):
        if isinstance(data, List):
            return [d.tolist() for d in data]
        return data.tolist()

mlup/ml/data_transformers/torch_tensor_data_transformer.py
nxexox/pymlup
class EmptyModel:
    def predict(self, *args, **kwargs):
        pass

mlup/ml/empty.py
nxexox/pymlup
import asyncio
import gc
import logging
import sys
from asyncio.locks import Lock as AsyncLock
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field, InitVar
from enum import Enum
from functools import partial
from multiprocessing import Lock
from typing import List, Dict, Callable, Optional, Any, Union

from mlup.config import set_logging_settings, LOGGING_CONFIG
from mlup.constants import (
    DEFAULT_X_ARG_NAME,
    ModelDataTransformerType,
    ModelLibraryType,
    BinarizationType,
    IS_X,
    StorageType, THERE_IS_ARGS, LoadedFile,
)
from mlup.ml.data_transformers.base import BaseDataTransformer
from mlup.errors import ModelLoadError, PredictError, PredictTransformDataError
from mlup.utils.loop import shutdown_pool_executor
from mlup.utils.profiling import TimeProfiler
from mlup.utils.interspection import analyze_method_params, get_class_by_path, auto_search_binarization_type


set_logging_settings(LOGGING_CONFIG, level=logging.INFO)
logger = logging.getLogger('mlup')


@dataclass
class ModelConfig:
    """
    MlupModel config class. This class have settings for model.

    # Information arguments
    name (str): Model name. Use for information about your model.
        Default is 'MyFirstMLupModel'.

    version (str): Model version. Use for information about your model.
        Default is '1.0.0.0'.

    # Model interface settings
    type (ModelLibraryType): Model type for choose current code for load and run model.
        See mlup.constants.ModelLibraryType for details.
        Default is ModelLibraryType.SKLEARN.

    columns (Optional[List[Dict[str, str]]]): Columns description for model predict method.
        Format List[Dict], Example [{"name": "col1", "type": "str", "required": True, "default": None}]
        name - Column name;
        type - Column type in string: int, str, float, bool.
        required - bool and optional field. By default is True
        default - Any data and optional.
        If you not set columns, then columns validation should be False.

    predict_method_name (str): Name for predict method by your model.
        This param use for analyze predict method and run predict method in your model.
        Default is "predict".

    # Model load settings
    auto_detect_predict_params (bool): If set is True, UP will parse predict method your model
        and create from predict method arguments, validators and ApiScheme for inner data.
        If set is False, then UP wait json with {data_for_predict: [..data for predict..]}.
        Default is True.

    storage_type (StorageType): Type storage, where should the model be loaded from.
        All storages code there is in mlup.ml.storage package.
        This param use in load() method, by it search storage class and use for load model.
        Params for storage class, can set to storage_kwargs.
        If set StorageType.memory, you can set ml to construct, without load from storage.
        Default: StorageType.memory.

    storage_kwargs (Dict): Naming arguments for storage class from storage_type.
        Default: empty Dict.

    binarization_type (BinarizationType): Type binarization model method.
        Use for load binary model from storage. See mlup.constants.BinarizationType for details.
        If set "auto", run auto search binarization type by model raw binary data and model name in storage.
        If binarization_type="auto" found binarization, than set binarization_type to found binarizer class.
        If binarization_type="auto" not found binarization, than raise ModelLoadError.
        Default is "auto".

    # Model work settings
    use_thread_loop (bool): Use concurrent.futures.ThreadPoolExecutor for create workers
        and run in workers pool predict in model.
        This useful for not blocking App CPU bound operations on the predict time.
        Default is True.

    max_thread_loop_workers (Optional[int]): Count thread workers in concurrent.futures.ThreadPoolExecutor.
        Param use if use_thread_loop is set True. If not set this param, then will be calculate,
        how calculate concurrent library - min(32, os.cpu_count() + 4).
        Default is None, that is min(32, os.cpu_count() + 4).

    data_transformer_for_predict (ModelDataTransformerType): How data type need for model predict method.
        See mlup.constants.ModelDataTransformerType for details.
        Default ModelDataTransformerType.NUMPY_ARR.

    data_transformer_for_predicted (ModelDataTransformerType): How data type model returned from predict method.
        See mlup.constants.ModelDataTransformerType for details.
        Default ModelDataTransformerType.NUMPY_ARR.

    dtype_for_predict (Optional[str]): Dtype for data_transformer_for_predict.
        Each data_transformer uses its own way of searching for a dtype by the specified one.
        But as a rule it is getattr([pandas, numpy, torch, tensorflow], dtype).
        The default is None, and the library itself determines which type.

    """
    name: str = 'MyFirstMLupModel'
    version: str = '1.0.0.0'

    # Model Interface settings
    type: Union[str, ModelLibraryType] = ModelLibraryType.SKLEARN
    columns: Optional[List[Dict[str, str]]] = None
    predict_method_name: str = 'predict'

    # Model load settings
    auto_detect_predict_params: bool = True
    storage_type: Union[str, StorageType] = StorageType.memory
    storage_kwargs: Dict = field(default_factory=dict, repr=False)
    binarization_type: Union[str, BinarizationType] = "auto"

    # Model work settings
    use_thread_loop: bool = True
    max_thread_loop_workers: Optional[int] = None
    data_transformer_for_predict: Union[str, ModelDataTransformerType] = ModelDataTransformerType.NUMPY_ARR
    data_transformer_for_predicted: Union[str, ModelDataTransformerType] = ModelDataTransformerType.NUMPY_ARR
    dtype_for_predict: Optional[str] = None

    def ml_dict(self):
        res = {}
        for n, f in ModelConfig.__dataclass_fields__.items():
            if f.repr is True:
                v = getattr(self, n)
                if isinstance(v, Enum):
                    v = v.value
                res[n] = v
        return res

    def ml_str(self, need_spaces: bool = False):
        res = []
        space = '    ' if need_spaces else ''
        for n, f in ModelConfig.__dataclass_fields__.items():
            if f.repr is True:
                v = getattr(self, n)
                if isinstance(v, Enum):
                    v = v.value
                res.append(space + f'{n}={v}')
        return '\n'.join(res)


@dataclass(repr=True)
class MLupModel(ModelConfig):
    """This is main UP model class.
    Create object UP with your ML model, set your settings and run your app.

    Args:
        ml (Any): Model object. Use for create UP from memory model, without UP load procedures.
            Recommended use for check work and configure your app. This argument set model_obj attribute.
        conf (ModelConfig): Config for model. If not set this class, config will create automatic with default params.

    """
    ml_model: InitVar[Any] = field(default=None, repr=False)
    conf: ModelConfig = field(default_factory=ModelConfig, repr=False)

    # Inner model data
    x_column_name: str = field(init=False, default=None)
    _model_obj = None
    _lock: Lock = field(init=False, default=None, repr=False)
    _async_lock: AsyncLock = field(init=False, default=None, repr=False)
    _pool_workers: ThreadPoolExecutor = field(init=False, default=None, repr=False)
    _predict_method: Callable = field(init=False, default=None, repr=False)
    _predict_arguments: List[Dict] = field(init=False, default=None, repr=False)
    _data_transformer_for_predict: BaseDataTransformer = field(init=False, default=None, repr=False)
    _data_transformer_for_predicted: BaseDataTransformer = field(init=False, default=None, repr=False)

    @property
    def model_obj(self):
        """Source model object."""
        if not self.loaded:
            raise ModelLoadError('Model object not found. Please call load().')
        return self._model_obj

    @property
    def loaded(self) -> bool:
        return self._model_obj is not None

    def __del__(self):
        """Remove ThreadPoolExecutor workers"""
        if self._pool_workers is not None:
            shutdown_pool_executor(self._pool_workers, wait=False, cancel_futures=False)

    def __getstate__(self):
        """Before pickle object"""
        logger.info(f'Running binarization {self}.')
        attributes = self.__dict__.copy()
        attributes.pop('_lock', None)
        attributes.pop('_async_lock', None)
        attributes.pop('_pool_workers', None)
        attributes.pop('_predict_method', None)
        attributes.pop('_predict_arguments', None)
        attributes.pop('x_column_name', None)
        return attributes

    def __setstate__(self, state):
        """After unpickle object"""
        logger.info(f'Running an {self} load from binary data.')
        self.__dict__ = state
        self._async_lock = AsyncLock()
        self._lock = Lock()
        if self.loaded:
            self.load_model_settings()

    def __post_init__(self, ml_model: Optional[Callable] = None):
        """Running custom construct code after call dataclass construct code"""
        if self._async_lock is None:
            self._async_lock = AsyncLock()
        if self._lock is None:
            self._lock = Lock()

        if self.conf is None:
            self.conf = ModelConfig()

        # Settings model from memory
        if StorageType(self.conf.storage_type) == StorageType.memory:
            if not ml_model:
                raise ModelLoadError(f'If you use {self.conf.storage_type}, need set "ml" argument.')

            self.conf.storage_kwargs['model'] = ml_model

    # Load model interface
    def _get_predict_method(self):
        try:
            return getattr(self.model_obj, self.conf.predict_method_name)
        except AttributeError as error:
            raise ModelLoadError(str(error))

    def _get_x_column_name(self) -> str:
        if self.conf.auto_detect_predict_params:
            for col_conf in self._predict_arguments:
                if col_conf.get(IS_X, False) or col_conf.get(THERE_IS_ARGS, False):
                    return col_conf['name']
        return DEFAULT_X_ARG_NAME

    # Work model interface
    def _get_data_transformer_for_predict(self) -> BaseDataTransformer:
        return get_class_by_path(self.conf.data_transformer_for_predict)(
            dtype_name=self.conf.dtype_for_predict
        )

    def _get_data_transformer_for_predicted(self) -> BaseDataTransformer:
        return get_class_by_path(self.conf.data_transformer_for_predicted)(
            dtype_name=self.conf.dtype_for_predict
        )

    def _transform_data_for_predict(self, src_data: List[Union[Dict, List]]):
        logger.debug(f'Transform data {len(src_data)} rows to model format.')
        with TimeProfiler('The processing data:', log_level='debug'):
            try:
                processing_data = self._data_transformer_for_predict.transform_to_model_format(
                    src_data,
                    self.conf.columns,
                )
            except Exception as e:
                logger.exception(f'Fail transform data rows {len(src_data)} to model format.')
                raise PredictTransformDataError(str(e))
            finally:
                logger.debug(f'Finish transform data {len(src_data)} rows to model format.')
        return processing_data

    def _transform_predicted_data(self, predicted_data: Any):
        logger.debug('Transform predicted data to response format.')
        with TimeProfiler('The processing data:', log_level='debug'):
            try:
                processing_data = self._data_transformer_for_predicted.transform_to_json_format(predicted_data)
            except Exception as e:
                logger.exception('Fail transform predicted data to response format.')
                raise PredictTransformDataError(str(e))
            finally:
                logger.debug('Finish transform predicted data to response format.')
        return processing_data

    def _prepare_predict_method(self, **other_predict_args):
        if other_predict_args:
            return partial(self._predict_method, **other_predict_args)
        return self._predict_method

    async def _predict(self, data_for_predict: Optional[Any] = None, **other_predict_args):
        logger.debug('The model processing the input data.')
        error = None
        loop = asyncio.get_running_loop()
        _predict_args = (data_for_predict,) if data_for_predict is not None else tuple()

        with TimeProfiler('The model work:'):
            try:
                if self.conf.use_thread_loop:
                    logger.debug(f'Running predict in ThreadPoolExecutor({self.conf.max_thread_loop_workers}).')
                    async with self._async_lock:
                        result = await loop.run_in_executor(
                            self._pool_workers,
                            self._prepare_predict_method(**other_predict_args),
                            *_predict_args
                        )
                else:
                    logger.debug('Running sync predict.')
                    with self._lock:
                        result = self._prepare_predict_method(**other_predict_args)(*_predict_args)
            except Exception as e:
                error = e

        if error:
            raise error

        return result

    def load(self, force_loading: bool = False):
        """
        Load binary model.
        If model already loaded, new load will not run. For force loading, set force_loading=True.

        :param bool force_loading: Reloading model, if model already loaded.

        """
        logger.info('Run load model.')
        if self._model_obj and not force_loading:
            raise ModelLoadError('Model is already loaded. For reload use force_reloading=True.')

        logger.debug(f'Load model with settings:\n{self.conf.ml_str(need_spaces=True)}')
        storage_class = get_class_by_path(self.conf.storage_type)
        logger.debug(f'Create storage {storage_class}.')
        storage = storage_class(**self.conf.storage_kwargs)

        with TimeProfiler('Time to load binary model to memory:'):
            try:
                loaded_files: list[LoadedFile] = storage.load()
            except Exception as e:
                raise ModelLoadError(str(e))

        logger.debug(f'Size raw models loaded data: {sys.getsizeof(loaded_files, default=-1)}')

        if StorageType(self.conf.storage_type) == StorageType.memory:
            self._model_obj = loaded_files[0].raw_data
        else:
            logger.debug(f'Search binarizer: "{self.conf.binarization_type}".')
            if self.conf.binarization_type == 'auto':
                binarization_type = auto_search_binarization_type(loaded_files[0])
                if binarization_type is not None:
                    self.conf.binarization_type = binarization_type
                else:
                    raise ModelLoadError(
                        'Binarizer with auto mode not found. Set binarizer manual or change your models.'
                    )
            binarization_class = get_class_by_path(self.conf.binarization_type)
            self._model_obj = binarization_class.deserialize(loaded_files[0])

        logger.info(f'Size deserializing models: {sys.getsizeof(self._model_obj, default=-1)}')

        del loaded_files
        gc.collect()
        self.load_model_settings()

    def load_model_settings(self):
        """Analyze binary model and make config for use model."""
        logger.info('Run load model settings.')
        if not self.loaded:
            raise ModelLoadError('Model not loaded to memory. Analyze impossible. Please call ml.load().')

        with TimeProfiler('Time to load model settings:'):
            self._data_transformer_for_predict = self._get_data_transformer_for_predict()
            self._data_transformer_for_predicted = self._get_data_transformer_for_predicted()
            self._predict_method = self._get_predict_method()
            self._predict_arguments = analyze_method_params(self._predict_method, self.conf.auto_detect_predict_params)
            self.x_column_name = self._get_x_column_name()
            if self.conf.use_thread_loop:
                self._pool_workers = ThreadPoolExecutor(
                    self.conf.max_thread_loop_workers, thread_name_prefix='mlup_predict'
                )
            else:
                self._pool_workers = None

    def get_X_from_predict_data(self, predict_data: Dict, remove: bool = True) -> Optional[List[Union[Dict, List]]]:
        """Get X param and value from predict data by model settings"""
        if not self.loaded:
            raise ModelLoadError('Model object not loaded. Please call load().')
        if remove:
            return predict_data.pop(self.x_column_name, None)
        return predict_data.get(self.x_column_name, None)

    async def predict(self, **predict_data):
        """
        Call model predict from data with python types.
        Example: up.ml.predict(X=[[1, 2, 3], [4, 5, 6]])

        """
        if not self.loaded:
            raise ModelLoadError('Model object not loaded. Please call load().')
        try:
            src_x = self.get_X_from_predict_data(predict_data)
            data_for_predict = None
            if src_x is not None:
                data_for_predict = self._transform_data_for_predict(src_x)

            predicted_data = await self._predict(data_for_predict, **predict_data)
            return self._transform_predicted_data(predicted_data)
        except PredictTransformDataError:
            raise
        except Exception as e:
            raise PredictError(e)

    async def predict_from(self, **predict_data):
        """
        Call model predict from data with library types.
        Example: up.ml.predict_from(X=numpy.array([[1, 2, 3], [4, 5, 6]]))

        """
        if not self.loaded:
            raise ModelLoadError('Model object not loaded. Please call load().')
        try:
            src_x = self.get_X_from_predict_data(predict_data)
            predicted_data = await self._predict(src_x, **predict_data)
            return self._transform_predicted_data(predicted_data)
        except PredictTransformDataError:
            raise
        except Exception as e:
            raise PredictError(e)

mlup/ml/model.py
nxexox/pymlup
import abc
from typing import Union, List, Callable

from mlup.constants import LoadedFile


class BaseStorage(metaclass=abc.ABCMeta):

    @abc.abstractmethod
    def load_bytes_single_file(self, *args, **kwargs) -> Union[str, bytes, Callable]:
        pass

    @abc.abstractmethod
    def load(self) -> List[LoadedFile]:
        pass

mlup/ml/storage/base.py
nxexox/pymlup
import logging
import os
import re
from dataclasses import dataclass
from typing import Union, List

from mlup.ml.storage.base import BaseStorage
from mlup.constants import LoadedFile


logger = logging.getLogger('mlup')


@dataclass
class DiskStorage(BaseStorage):
    path_to_files: str
    files_mask: Union[str, re.Pattern] = re.compile(r'(\w.-_)*.pckl')
    need_load_file: bool = False

    def load_bytes_single_file(self, path_to_file: str) -> LoadedFile:
        logger.info(f'Load file {path_to_file}')
        if self.need_load_file and os.path.isfile(path_to_file):
            with open(path_to_file, 'rb') as f:
                return LoadedFile(f.read(), path=path_to_file)
        logger.debug(f'Return path {path_to_file} to binarizer')
        return LoadedFile(path=path_to_file)

    def load(self) -> List[LoadedFile]:
        logger.debug(f'Start analyze data path {self.path_to_files} by mask {self.files_mask}')
        if os.path.isfile(self.path_to_files):
            logger.debug(f'Found single file by path {self.path_to_files}')
            if re.search(self.files_mask, str(self.path_to_files)):
                return [self.load_bytes_single_file(self.path_to_files)]
            raise FileNotFoundError(f'File {self.path_to_files} does not match mask {self.files_mask}')

        file_pathes = [
            os.path.join(self.path_to_files, f_name)
            for f_name in os.listdir(self.path_to_files)
            if re.search(self.files_mask, f_name)
        ]
        logger.debug('Found files for loading: ' + ', '.join(file_pathes))
        return [self.load_bytes_single_file(f_path) for f_path in file_pathes]

mlup/ml/storage/local_disk.py
nxexox/pymlup
from dataclasses import dataclass
from typing import List, Callable

from mlup.ml.storage.base import BaseStorage
from mlup.constants import LoadedFile


@dataclass
class MemoryStorage(BaseStorage):
    model: Callable

    def load_bytes_single_file(self, *args, **kwargs) -> Callable:
        return self.model

    def load(self) -> List[LoadedFile]:
        return [LoadedFile(self.model)]

mlup/ml/storage/memory.py
nxexox/pymlup
import logging.config
from dataclasses import field, dataclass, InitVar
from pathlib import Path
from typing import Dict, Any, Optional, Union

from mlup.config import ConfigProvider, LOGGING_CONFIG, set_logging_settings
from mlup.ml.empty import EmptyModel
from mlup.ml.model import MLupModel, ModelConfig
from mlup.utils.loop import run_async
from mlup.web.app import MLupWebApp, WebAppConfig


set_logging_settings(LOGGING_CONFIG, level=logging.INFO)
logger = logging.getLogger('mlup')


def generate_default_config(path_to_file: Optional[str] = None) -> Optional[Dict]:
    up = UP(ml_model=EmptyModel())
    if not path_to_file:
        return up.to_dict()
    if path_to_file.endswith('json'):
        up.to_json(path_to_file)
    else:
        up.to_yaml(path_to_file)


@dataclass
class Config(ModelConfig, WebAppConfig):
    pass


@dataclass(repr=False)
class UP:
    """This is main UP class.
    Create object UP with your ML model, set your settings and run your web app.

    """
    ml_model: InitVar[Any] = None
    conf: Config = None

    # Inner model data
    config_provider: ConfigProvider = field(init=False, default=None, repr=False)
    ml: MLupModel = field(init=False, default=None, repr=True)
    _wep_app: MLupWebApp = field(init=False, default=None, repr=True)

    def __post_init__(self, ml_model: Any):
        """Running custom construct code after call dataclass construct code"""
        if self.config_provider is None:
            self.config_provider = ConfigProvider(
                obj_for_config=self,
                configs_objects_map={"ml": ModelConfig, "web": WebAppConfig}
            )
        if self.conf is None:
            self.conf = Config()

        if isinstance(ml_model, MLupModel):
            self.ml = ml_model
            self.ml.conf = self.conf
        else:
            self.ml = MLupModel(
                ml_model=ml_model,
                conf=self.conf,
            )

    @property
    def web(self) -> MLupWebApp:
        """MLupWebApp object"""
        if self._wep_app is None:
            self._wep_app = MLupWebApp(ml=self.ml, conf=self.conf)
        return self._wep_app

    @classmethod
    def load_from_dict(cls, conf: Dict, load_model: bool = True):
        """
        Load UP object from dict.

        :param Dict conf: Conf dict for loading UP object.
        :param bool load_model: Need load model to memory from storage? Default is True.

        """
        up = cls(ml_model=EmptyModel())
        up.config_provider.load_from_dict(conf)
        if load_model:
            up.ml.load()
        return up

    def to_dict(self):
        """UP config to dict"""
        return self.config_provider.get_config_dict()

    @classmethod
    def load_from_json(cls, conf_path: Union[str, Path], load_model: bool = True):
        """
        Load UP object from json config.

        :param Union[str, Path] conf_path: Path to JSON file with config.
        :param bool load_model: Need load model to memory from storage? Default is True.

        """
        up = cls(ml_model=EmptyModel())
        up.config_provider.load_from_json(conf_path)
        if load_model:
            up.ml.load()
        return up

    def to_json(self, path_to_file: Union[str, Path]):
        """
        Save up config to JSON file.

        :param Union[str, Path] path_to_file: Path to result config file.

        """
        return self.config_provider.save_to_json(path_to_file)

    @classmethod
    def load_from_yaml(cls, conf_path: Union[str, Path], load_model: bool = True):
        """
        Load UP object from yaml config.

        :param Union[str, Path] conf_path: Path to YAML file with config.
        :param bool load_model: Need load model to memory from storage? Default is True.

        """
        up = cls(ml_model=EmptyModel())
        up.config_provider.load_from_yaml(conf_path)
        if load_model:
            up.ml.load()
        return up

    def to_yaml(self, path_to_file: Union[str, Path]):
        """
        Save up config to YAML file.

        :param Union[str, Path] path_to_file: Path to result config file.

        """
        return self.config_provider.save_to_yaml(path_to_file)

    def predict(self, **predict_data):
        """
        Call model predict.

        :param predict_data: Data for predict. This data have same format as predict from web app.
        Example:
            up.predict(X=[[1, 2, 3], [4, 5, 6]])

        """
        return run_async(self.ml.predict, **predict_data)

    def predict_from(self, **predict_data):
        """
        Call model predict without transform data before model predict.

        :param predict_data: Data for predict with ML format, use numpy, pandas and etc libraries.
        Example:
            up.predict(X=numpy.array([[1, 2, 3], [4, 5, 6]]))

        """
        return run_async(self.ml.predict_from, **predict_data)

    async def async_predict(self, **predict_data):
        """
        Call model predict without use mlup.utils.loop.run_async.

        :param predict_data: Data for predict. This data have same format as predict from web app.
        Example:
            await up.predict(X=[[1, 2, 3], [4, 5, 6]])

        """
        return await self.ml.predict(**predict_data)

    def run_web_app(self, daemon: bool = False, force_load: bool = False):
        """
        Run web app with ML model.

        :param bool daemon: Run web app with daemon mode and unblock terminal. Default is False.
        :param bool force_load: Reload web_app before start, even web_app was loaded.

        """
        if not self.web.loaded or force_load:
            self.web.load()
        self.web.run(daemon=daemon)

    def stop_web_app(self, shutdown_timeout: Optional[float] = None):
        """Stop web app, if web app was runned in daemon mode."""
        self.web.stop(shutdown_timeout=shutdown_timeout)

mlup/up.py
nxexox/pymlup
from collections import OrderedDict
from threading import RLock
import time


class TTLOrderedDict(OrderedDict):
    """
    OrderedDict with TTL
    Extra args and kwargs are passed to initial .update() call
    """
    def __init__(self, default_ttl, *args, **kwargs):
        """
        Be warned, if you use this with Python versions earlier than 3.6
        when passing **kwargs order is not preseverd.
        """
        assert isinstance(default_ttl, int)
        self._default_ttl = default_ttl
        self._lock = RLock()
        super().__init__()
        self.update(*args, **kwargs)

    def __repr__(self):
        return '<TTLOrderedDict@%#08x; ttl=%r, OrderedDict=%r;>' % (
            id(self), self._default_ttl, self.items())

    def __len__(self):
        with self._lock:
            self._purge()
            return super().__len__()

    def set_ttl(self, key, ttl, now=None):
        """Set TTL for the given key"""
        if now is None:
            now = time.time()
        with self._lock:
            value = self[key]
            super().__setitem__(key, (now + ttl, value))

    def get_ttl(self, key, now=None):
        """Return remaining TTL for a key"""
        if now is None:
            now = time.time()
        with self._lock:
            expire, _value = super().__getitem__(key)
            return expire - now

    def expire_at(self, key, timestamp):
        """Set the key expire timestamp"""
        with self._lock:
            value = self.__getitem__(key)
            super().__setitem__(key,  (timestamp, value))

    def is_expired(self, key, now=None):
        """ Check if key has expired, and return it if so"""
        with self._lock:
            if now is None:
                now = time.time()

            expire, _value = super().__getitem__(key)

            if expire:
                if expire < now:
                    return key

    def _purge(self):
        _keys = list(super().__iter__())
        _remove = [key for key in _keys if self.is_expired(key)]  # noqa
        [self.__delitem__(key) for key in _remove]

    def __iter__(self):
        """
        Yield only non expired keys, without purging the expired ones
        """
        with self._lock:
            for key in super().__iter__():
                if not self.is_expired(key):
                    yield key

    def __setitem__(self, key, value):
        with self._lock:
            if self._default_ttl is None:
                expire = None
            else:
                expire = time.time() + self._default_ttl
            super().__setitem__(key,  (expire, value))

    def __delitem__(self, key):
        with self._lock:
            super().__delitem__(key)

    def __getitem__(self, key):
        with self._lock:
            if self.is_expired(key):
                self.__delitem__(key)
                raise KeyError
            item = super().__getitem__(key)[1]
            return item

    def keys(self):
        with self._lock:
            self._purge()
            return super().keys()

    def items(self):
        with self._lock:
            self._purge()
            _items = list(super(OrderedDict, self).items())
            return [(k, v[1]) for (k, v) in _items]

    def values(self):
        with self._lock:
            self._purge()
            _values = list(super(OrderedDict, self).values())
            return [v[1] for v in _values]

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

mlup/utils/collections.py
nxexox/pymlup
import importlib
import inspect
import logging
from enum import Enum
from typing import Callable, Any, Union, Optional, Type

from mlup.constants import IS_X, THERE_IS_ARGS, DEFAULT_X_ARG_NAME, BinarizationType, LoadedFile
from mlup.utils.profiling import TimeProfiler


logger = logging.getLogger('mlup')


def get_class_by_path(path_to_class: Union[str, Enum]) -> Any:
    """
    Get class by path to class. Use importlib.import_module.

    :param Union[str, Enum] path_to_class: Path to class for import and return.

    """
    if isinstance(path_to_class, Enum):
        path_to_class = path_to_class.value

    # Try get class by path
    try:
        return import_obj(path_to_class)
    except (ModuleNotFoundError, AttributeError) as e:
        logger.warning(f'Object {path_to_class} import error. {e}')
        raise


def import_obj(path_to_import_obj: str) -> Any:
    """
    Import any python object.

    :param str path_to_import_obj: Path for import object.

    :return: Imported object.

    """
    mod = importlib.import_module(path_to_import_obj.rsplit('.', 1)[0])
    return getattr(mod, path_to_import_obj.rsplit('.', 1)[1])


def analyze_method_params(func: Callable, auto_detect_predict_params: bool = True, ignore_self: bool = True):
    """
    Parse method and create dict with method parameters.

    :param Callable func: Func for parse arguments.
    :param bool auto_detect_predict_params: Auto detect predict params and set List to first param.
    :param bool ignore_self: Ignore first param with name self and cls.

    :return: Dict with parsing func arguments.
    :rtype: Dict[str, Dict[str, str]]

    def example(a, b = 100, *, c: float = 123):
        pass

    method_params = analyze_method_params(method)
    method_params
    [{'name': 'a', 'required': True},
     {'name': 'b', 'required': False, 'default': 100},
     {'name': 'c', 'type': float, 'required': False, 'default': 123}]

    """
    sign = inspect.signature(func)
    arg_spec = inspect.getfullargspec(func)
    result = []
    types = {
        int: 'int',
        float: 'float',
        bool: 'bool',
        str: 'str',
    }
    is_there_args = False
    logger.info(f'Analyzing arguments in {func}.')

    _found_X = False
    for i, (param_name, param_obj) in enumerate(sign.parameters.items()):
        if arg_spec.varargs == param_name:
            is_there_args = True
        if arg_spec.varargs == param_name or arg_spec.varkw == param_name:
            logger.warning(f'Found unpacked argument: {arg_spec}. Skip argument')
            continue

        # Skip first arg if it's self, cls
        if i == 0 and ignore_self:
            if param_name.lower() in {'self', 'cls'}:
                continue

        param_data = {
            'name': param_name,
            'required': True,
        }
        if param_obj.annotation is not inspect._empty:
            if param_obj.annotation in types:
                param_data['type'] = types[param_obj.annotation]
            else:
                logger.warning(
                    f'For model predict argument writes not supported type {param_obj.annotation}. '
                    f'Skip added validation'
                )

        if param_obj.default is not inspect._empty:
            param_data['required'] = False
            param_data['default'] = param_obj.default
            if 'type' not in param_data and type(param_obj.default) in types:
                param_data['type'] = types[type(param_obj.default)]

        if param_name.lower().strip() == 'x' and auto_detect_predict_params:
            logger.info('Found X param in model params. Set List type')
            param_data['type'] = 'List'
            param_data[IS_X] = True
            _found_X = True

        result.append(param_data)

    if not _found_X and auto_detect_predict_params:
        if result:
            logger.info(
                f'X argument in predict method not found. '
                f'For predict data use first argument with name "{result[0]["name"]}".'
            )
            result[0]['type'] = 'List'
            result[0][IS_X] = True
        else:
            logger.info('Not found arguments in predict method.')

    if not result and is_there_args:
        logger.info(
            f'Found not naming args in predict method. For call predict, use "{DEFAULT_X_ARG_NAME}" X key. '
            f'Example: mlup_model.predict({DEFAULT_X_ARG_NAME}=[obj_for_pred_1, obj_for_pred_2, ...]).'
        )
        result.append({'name': DEFAULT_X_ARG_NAME, 'required': False, THERE_IS_ARGS: True})
    return result


def auto_search_binarization_type(loaded_file: LoadedFile) -> Optional[Type[BinarizationType]]:
    """
    Search binarizer by model raw binary data and model name.
    Search in only mlup binarizers from mlup.constants.BinarizationType.

    :param LoadedFile loaded_file: Model raw data information.

    :return: Found binarizer class if found or None if not found.
    :rtype: Optional[Type[BinarizationType]]

    """
    logger.info('Run auto search binarizer.')
    probabilities = []
    with TimeProfiler('Time to auto search binarizer:', log_level='info'):
        for binarizer_path in BinarizationType:
            try:
                binarization_class = get_class_by_path(binarizer_path.value)
            except (ModuleNotFoundError, AttributeError):
                continue

            probability = binarization_class.is_this_type(loaded_file=loaded_file)
            probabilities.append((probability, binarizer_path))
            logger.debug(f'Binarizer "{binarizer_path}" have probability {probability}.')

        high_prob = sorted(probabilities, key=lambda x: x[0])[-1]
        logger.debug(f'Hypothesis binarizer "{high_prob[1]}" with probaility {high_prob[0]}.')
        if high_prob[0] > 0.0:
            logger.info(f'Found binarizer "{high_prob[1]}" with probaility {high_prob[0]}.')
            return high_prob[1]

    logger.info('Binarizer not found.')
    return None

mlup/utils/interspection.py
nxexox/pymlup
import logging
from typing import Optional

from uvicorn.logging import DefaultFormatter

from mlup.utils.oop import MetaSingleton


class SwitchFormatter(DefaultFormatter, metaclass=MetaSingleton):
    def __init__(self, fmt: Optional[str] = None, *args, **kwargs):
        self._fmts = {'default': fmt}
        kwargs_without_fmt = {}
        for arg, val in kwargs.items():
            if arg.startswith('fmt_'):
                self._fmts[arg.split('_', 1)[1]] = val
            else:
                kwargs_without_fmt[arg] = val

        super().__init__(fmt=fmt, *args, **kwargs_without_fmt)

    def set_fmt(self, fmt_name: str = 'default'):
        fmt = self._fmts[fmt_name]
        self._style = logging.PercentStyle(fmt)
        # This "if" for python3.7-
        if hasattr(self._style, 'validate'):
            self._style.validate()
        self._fmt = self._style._fmt


def configure_logging_formatter(formatter_name: str = 'default'):
    SwitchFormatter().set_fmt(formatter_name)

mlup/utils/logging.py
nxexox/pymlup
import asyncio
import sys
import threading
from concurrent.futures import Executor


class RunThreadCoro(threading.Thread):
    def __init__(self, async_func, args, kwargs):
        self.async_func = async_func
        self.args = args
        self.kwargs = kwargs
        self.result = None
        super().__init__()

    def run(self):
        self.result = asyncio.run(self.async_func(*self.args, **self.kwargs))


def run_async(func, *args, **kwargs):
    # If loop is running, asyncio.run raise:
    # RuntimeError: asyncio.run() cannot be called from a running event loop
    # https://stackoverflow.com/questions/55409641/asyncio-run-cannot-be-called-from-a-running-event-loop-when-using-jupyter-no
    try:
        loop = asyncio.get_running_loop()
    except RuntimeError:
        loop = None
    if loop and loop.is_running():
        thread = RunThreadCoro(func, args, kwargs)
        thread.start()
        thread.join()
        return thread.result
    else:
        return asyncio.run(func(*args, **kwargs))


def create_async_task(coro, *, name=None):
    task_kwargs = {}
    if sys.version_info.minor >= 8:
        task_kwargs['name'] = name
    return asyncio.create_task(coro, **task_kwargs)


def shutdown_pool_executor(pool_executor: Executor, wait: bool = False, cancel_futures: bool = False):
    kwargs = {}
    if sys.version_info.minor >= 9:
        kwargs['cancel_futures'] = cancel_futures
    pool_executor.shutdown(wait, **kwargs)

mlup/utils/loop.py
nxexox/pymlup

class MetaSingleton(type):
    _instances = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super().__call__(*args, **kwargs)
        return cls._instances[cls]

    @classmethod
    def get_singleton_instance(cls):
        return cls._instances.get(cls, None)

mlup/utils/oop.py
nxexox/pymlup
import time
from typing import Optional

from mlup.config import logger


class TimeProfiler:
    _start: float
    _end: float

    def __init__(self, msg: Optional[str] = None, log_level: str = 'info'):
        self.msg = msg
        self.logger = getattr(logger, log_level)

    def __enter__(self):
        self._start = time.monotonic()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._end = time.monotonic()
        msg = 'Time to func'
        if self.msg:
            msg = self.msg
        self.logger(
            f'{msg} {self._end - self._start:.3f}.'
        )

mlup/utils/profiling.py
nxexox/pymlup
from typing import List, Dict, Tuple

from fastapi import FastAPI
from fastapi.openapi.utils import get_openapi

from mlup.constants import DEFAULT_X_ARG_NAME, IS_X
from mlup.ml.model import MLupModel


_openapi_types_map = {
    "float": "number",
    "int": "integer",
    "str": "string",
    "bool": "boolean",
    "list": "array",
}


def _fix_scheme_by_paths(point: Dict, response_codes: List[str]):
    for http_method_name, http_method_val in point.items():
        # Remove common codes
        for resp_code in response_codes:
            if resp_code in http_method_val["responses"]:
                del http_method_val["responses"][resp_code]
        # Fix method in operationId, if handler have many http methods.
        if not http_method_val['operationId'].endswith(http_method_name.lower()):
            http_method_val['operationId'] = (
                http_method_val['operationId'].rsplit('_', 1)[0] + f'_{http_method_name.lower()}'
            )

    return point


def make_columns_object_openapi_scheme(src_columns: List[Dict]) -> Tuple[Dict, List]:
    cols_openapi_config = {}
    required_columns = []
    for col_config in src_columns:
        col_name, col_type = col_config["name"], col_config.get("type", "str")
        col_required, col_default = col_config.get("required", True), col_config.get("default", None)

        _col_config = {"type": _openapi_types_map[col_type.lower()]}
        title = []
        if col_default is not None:
            title.append("Default")
            _col_config["default"] = col_default
        if col_required:
            required_columns.append(col_name)
        else:
            title.append("Optional")

        title.append(col_type.capitalize())

        if col_config.get(IS_X, False):
            _col_config["items"] = {"$ref": "#/components/schemas/ColumnsForPredict"}
        cols_openapi_config[col_name] = {"title": ' '.join(title), **_col_config}

    return cols_openapi_config, required_columns


def generate_openapi_schema(app: FastAPI, ml: MLupModel):
    """
    openapi_schema["info"] = {
        "title": DOCS_TITLE,
        "version": DOCS_VERSION,
        "description": "Learn about programming language history!",
        "termsOfService": "http://programming-languages.com/terms/",
        "contact": {
            "name": "Get Help with this API",
            "url": "http://www.programming-languages.com/help",
            "email": "support@programming-languages.com"
        },
        "license": {
            "name": "Apache 2.0",
            "url": "https://www.apache.org/licenses/LICENSE-2.0.html"
        },
    }
    """
    # TODO: Optimize, dublicate code in api_validation.py
    openapi_schema = get_openapi(
        title=app.title,
        version=app.version,
        description=app.description,
        openapi_version=app.openapi_version,
        terms_of_service=app.terms_of_service,
        contact=app.contact,
        license_info=app.license_info,
        routes=app.routes,
        tags=app.openapi_tags,
        servers=app.servers,
    )

    if "/health" in openapi_schema["paths"]:
        openapi_schema["paths"]["/health"] = _fix_scheme_by_paths(
            openapi_schema["paths"]["/health"], ['422', '429', '500']
        )
    if "/info" in openapi_schema["paths"]:
        openapi_schema["paths"]["/info"] = _fix_scheme_by_paths(
            openapi_schema["paths"]["/info"], ['422', '429', '500']
        )

    openapi_schema["paths"]["/predict"] = {
      "post": {
        "summary": "Predict",
        "operationId": "predict_predict_post",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/PredictItems"
              }
            }
          },
          "required": True
        },
        "responses": {
          "200": {
            "description": "Successful Response",
            "content": {
              "application/json": {
                "schema": {}
              }
            }
          },
          '422': {
            'description': 'Error with validation input data',
            'content': {
              'application/json': {
                'schema': {'$ref': '#/components/schemas/ApiErrorResponse'}
              }
            }
          },
          '429': {
            'description': 'Throttling input request error',
            'content': {
              'application/json': {
                'schema': {'$ref': '#/components/schemas/ApiErrorResponse'}
              }
            }
          },
          '500': {
            'description': 'Error with predict process exception',
            'content': {
              'application/json': {
                'schema': {'$ref': '#/components/schemas/ApiErrorResponse'}
              }
            }
          }
        }
      }
    }

    cols_openapi_config, required_columns = {}, []
    if ml.conf.columns:
        cols_openapi_config, required_columns = make_columns_object_openapi_scheme(ml.conf.columns)

    predict_args_config = {
        DEFAULT_X_ARG_NAME: {
            "title": "Data for predict",
            "type": "array",
            "items": {"$ref": "#/components/schemas/ColumnsForPredict"},
        }
    }
    required_args = [DEFAULT_X_ARG_NAME]
    if ml.conf.auto_detect_predict_params:
        predict_args_config, required_args = make_columns_object_openapi_scheme(ml._predict_arguments)

    openapi_schema["components"]["schemas"]["PredictItems"] = {
        "title": "Predict items",
        "required": required_args,
        "type": "object",
        "properties": predict_args_config
    }
    openapi_schema["components"]["schemas"]["ColumnsForPredict"] = {
        "title": "Columns for predict",
        "required": required_columns,
        "type": "object",
        "properties": cols_openapi_config,
    }

    app.openapi_schema = openapi_schema
    return app.openapi_schema


def openapi_schema(app: FastAPI, ml: MLupModel):
    if not app.openapi_schema:
        app.openapi_schema = generate_openapi_schema(app, ml)
    return app.openapi_schema

mlup/web/api_docs.py
nxexox/pymlup
import logging
from dataclasses import dataclass
from typing import List, Union, Optional

from fastapi import Request, status
from fastapi.encoders import jsonable_encoder
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ValidationError

from mlup.constants import PREDICT_ID_HEADER
from mlup.errors import PredictError, PredictWaitResultError, PredictTransformDataError, \
    PredictValidationInnerDataError


logger = logging.getLogger('mlup')


class ApiError(BaseModel):
    loc: List
    msg: str
    type: str


class ApiErrorResponse(BaseModel):
    detail: List[ApiError]
    predict_id: Optional[str] = None


@dataclass
class ApiRequestError(Exception):
    message: str
    status_code: int = 400
    type: str = 'request_error'


def api_exception_handler(request: Request, exc: Union[ValidationError, ApiRequestError]):
    headers = {}
    error_res = ApiErrorResponse(detail=exc.errors())
    if hasattr(exc, 'predict_id'):
        error_res.predict_id = exc.predict_id
        headers[PREDICT_ID_HEADER] = exc.predict_id

    logger.debug(f'Response error. Status: {status.HTTP_422_UNPROCESSABLE_ENTITY}. Body: {error_res.dict()}.')

    return JSONResponse(
        content=jsonable_encoder(error_res.dict()),
        status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
        headers=headers,
    )


def api_request_error_handler(request: Request, exc: ApiRequestError):
    headers = {}
    error_res = ApiErrorResponse(detail=[{"loc": [], "msg": exc.message, "type": exc.type}])
    if hasattr(exc, 'predict_id'):
        error_res.predict_id = exc.predict_id
        headers[PREDICT_ID_HEADER] = exc.predict_id

    logger.debug(f'Response error. Status: {exc.status_code}. Body: {error_res.dict()}.')

    return JSONResponse(
        content=jsonable_encoder(error_res.dict()),
        status_code=exc.status_code,
        headers=headers,
    )


def predict_errors_handler(
    request: Request,
    exc: Union[PredictError, PredictWaitResultError, PredictTransformDataError, PredictValidationInnerDataError],
):
    headers = {}
    error_res = ApiErrorResponse(detail=[{"loc": [], "msg": str(exc), "type": exc.type}])
    if hasattr(exc, 'predict_id'):
        error_res.predict_id = exc.predict_id
        headers[PREDICT_ID_HEADER] = exc.predict_id

    logger.debug(f'Response error. Status: {exc.http_status}. Body: {error_res.dict()}.')

    return JSONResponse(
        content=jsonable_encoder(error_res.dict()),
        status_code=exc.http_status,
        headers=headers,
    )

mlup/web/api_errors.py
nxexox/pymlup
import logging
from typing import List, Any, Dict, Tuple, Optional, Type

from pydantic import BaseModel, create_model, Field

from mlup.constants import IS_X, DEFAULT_X_ARG_NAME
from mlup.ml.model import MLupModel


logger = logging.getLogger('mlup')


class MlModelPredictRequest(BaseModel):
    pass


def validator_by_required(v):
    if v is None:
        raise ValueError('Value is required')
    return v


def validator_by_int(v):
    if not isinstance(v, int):
        raise ValueError('Value not int type')
    return v


def validator_by_float(v):
    if not isinstance(v, float):
        raise ValueError('Value not float type')
    return v


def validator_by_bool(v):
    if not isinstance(v, bool):
        raise ValueError('Value not bool type')
    return v


def validator_by_str(v):
    if not isinstance(v, str):
        raise ValueError('Value not string type')
    return v


_type_validators = {
    'int': validator_by_int,
    'float': validator_by_float,
    'str': validator_by_str,
    'bool': validator_by_bool,
}


def make_map_pydantic_columns(
    src_columns: List[Dict[str, Any]],
    x_model: Optional[BaseModel] = Any,
) -> Tuple[Dict, Dict]:
    """
    Create Map columns for create pydantic model, by columns from config.
    Use for creating pydantic models in WebApp.

    Future:
        Example create custom validator:
        __validators__[f'{col_config["name"]}_required_validator'] = validator(
            col_config['name'], allow_reuse=True
        )(validator_by_required)

    :param List[Dict[str, Any]] src_columns: List columns from config.
    :param Optional[BaseModel] x_model: Col type from predict method in model.
        Use for auto detect X in model predict method.

    :return: Two dicts.
        The first dict is columns map for func pydantic.create_model.
        The second dict is columns validators for arg __validators__ in pydantic.create_model.
            Right now return empty Dict
    :rtype: Tuple[Dict, Dict]

    """
    column_types = {
        'int': int,
        'float': float,
        'bool': bool,
        'str': str,
        'list': list,
    }
    __validators__ = {}
    columns_pydantic_format = {}
    # If set None, from ml.columns
    src_columns = src_columns or []

    for col_config in src_columns:
        # Get col type and create col type validator.
        if col_config.get(IS_X, False):
            col_type = List[x_model]
        else:
            try:
                col_type = column_types[col_config['type'].lower()]
            except KeyError:
                logger.warning(
                    f'Field "{col_config["name"]}" has not a type. Please write type for field. '
                    f'Supported types {", ".join(column_types.keys())}.'
                )
                col_type = Any

        # Required
        field_required_default_value = Field(...)
        if 'default' in col_config:
            # With default
            field_required_default_value = Field(col_config['default'])
        elif col_config.get('required', True) is False:
            # Now required
            field_required_default_value = Field(default=None)
        field_required_default_value.title = col_config['name']

        # Make required validator
        columns_pydantic_format[col_config["name"]] = (col_type, field_required_default_value)

    return columns_pydantic_format, __validators__


def create_pydantic_predict_model(
    ml: MLupModel,
    column_validation: bool,
    custom_column_pydantic_model: Optional[Type[BaseModel]] = None,
) -> Type[MlModelPredictRequest]:
    # If model columns not validate, set Any
    mlup_columns_model = Any

    # If set custom column validation
    if custom_column_pydantic_model is not None:
        mlup_columns_model = custom_column_pydantic_model
    # If UP column validation IS on
    elif column_validation:
        columns_pydantic_format, __validators__ = make_map_pydantic_columns(ml.conf.columns)
        mlup_columns_model = create_model(
            "MLupModelColumns",
            __validators__=__validators__,
            **columns_pydantic_format,
        )

    # Make first arguments level
    if ml.conf.auto_detect_predict_params:
        pred_args_pydantic_format, __pred_args_validators__ = make_map_pydantic_columns(
            ml._predict_arguments,
            x_model=mlup_columns_model,
        )
        mlup_predict_pydantic_model = create_model(
            "MLupRequestPredictPydanticModel",
            __base__=MlModelPredictRequest,
            __validators__=__pred_args_validators__,
            **pred_args_pydantic_format,
        )
    else:
        mlup_predict_pydantic_model = create_model(
            "MLupRequestPredictPydanticModel",
            __base__=MlModelPredictRequest,
            **{DEFAULT_X_ARG_NAME: (List[mlup_columns_model], ...)}
        )
    return mlup_predict_pydantic_model

mlup/web/api_validators.py
nxexox/pymlup
import asyncio
import json
import logging
import threading
import time
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
from enum import Enum
from functools import partial, wraps
from typing import Type, Optional, Dict

from fastapi import FastAPI, Request as FastAPIRequest, Response as FastAPIResponse
from fastapi.exceptions import RequestValidationError
from pydantic import BaseModel as PydanticBaseModel, ValidationError
import uvicorn

from mlup.config import LOGGING_CONFIG
from mlup.constants import ITEM_ID_COL_NAME, WebAppArchitecture, PREDICT_ID_HEADER
from mlup.errors import PredictError, WebAppLoadError, PredictWaitResultError, PredictTransformDataError, \
    PredictValidationInnerDataError
from mlup.ml.model import MLupModel
from mlup.utils.crypto import generate_unique_id
from mlup.utils.interspection import get_class_by_path
from mlup.utils.logging import configure_logging_formatter
from mlup.utils.loop import run_async
from mlup.web.api_docs import openapi_schema
from mlup.web.api_errors import (
    ApiErrorResponse,
    api_exception_handler, ApiRequestError, predict_errors_handler, api_request_error_handler,
)
from mlup.web.api_validators import create_pydantic_predict_model, MlModelPredictRequest
from mlup.web.architecture.base import BaseWebAppArchitecture


logger = logging.getLogger('mlup')


def _requests_throttling(func):
    @wraps(func)
    async def wrap(self, *args, **kwargs):
        if not self.conf.throttling_max_requests:
            return await func(self, *args, **kwargs)

        if self._throttling_max_requests_current_count < self.conf.throttling_max_requests:
            self._throttling_max_requests_current_count += 1
            try:
                return await func(self, *args, **kwargs)
            finally:
                self._throttling_max_requests_current_count -= 1
        else:
            raise ApiRequestError(
                'Max requests in app. Please try again later.',
                status_code=429,
                type='throttling_error',
            )

    return wrap


def _set_predict_id_to_response_headers(func):
    @wraps(func)
    async def wrap(*args, response: FastAPIResponse, **kwargs):
        predict_id = generate_unique_id()
        response.headers[PREDICT_ID_HEADER] = predict_id
        try:
            return await func(*args, response=response, **kwargs)
        except Exception as e:
            e.predict_id = predict_id
            raise

    return wrap


@dataclass
class WebAppConfig:
    """
    WebAppConfig config class. This class have settings for web app.

    # WebApp web outer interface settings
    host (str): Web host for web app.
        Default is '0.0.0.0'.

    port (int): Web port for web app.
        Default is 8009.

    web_app_version (str): User version for web_app.
        Using the mlup version may not be nice to users your web app.
        However, you can change the library version without changing the model.
        And then the code of the web application backend actually changes.
        You can use this field to notify users of these changes.
        Default is '1.0.0.0'.

    column_validation (bool): Do I need to create a validator for columns from columns config
        that are sent by the user through the web application?
        Default is False.

    custom_column_pydantic_model (Optional[Type[PydanticBaseModel]]): Custom column pydantic model.
        Use for validation. If set this field, flag column_validation not use. Validation will be always.
        Default is None.

    # WebApp architecture settings
    mode (WebAppArchitecture): Work mode web app. We have three modes for work:
        directly_to_predict, worker_and_queue, batching.
        See mlup.constants.WebAppArchitecture for details.
        Default is WebAppArchitecture.directly_to_predict.

    max_queue_size (int): Max queue size with inner data for model.
        It's use in modes worker_and_queue and batching, for controlling queue inner requests size.
        Default is 100.

    ttl_predicted_data (int): Max time member predict result data.
        It's need for clean memory, if client not returned for predict result.
        Default is 60.

    ttl_client_wait (float): Max time wait results for clients, in single request.
        If the client has not waited for a response within this time, the server responds with a 408 response code
        and the user must make a second request.
        This is necessary so as not to make customers wait forever and all the ensuing consequences.
        Default is 30.0

    min_batch_len (int): Min batch len for start predict.
        Batching is regulated by two parameters: the batch accumulation time and the size of the accumulated batch.
        If one of the parameters has reached its limit, then the batch predictor is launched.
        It's min batch size for call batch predict.
        Default is 10.

    batch_worker_timeout (float): Max time for pending before run batching.
        Batching is regulated by two parameters: the batch accumulation time and the size of the accumulated batch.
        If one of the parameters has reached its limit, then the batch predictor is launched.
        It's max time waiting batch for call batch predict.
        Default is 1.0.

    is_long_predict (bool): Added get-predict api method and add return predict_id to predict response.
        Use only in worker_and_queue and batching modes.
        If set this param, after request to /predict, client get predict_id in response.
        If not set this param, than after ttl_client_wait time, client can't get predict result.
        Default is False.

    # WebApp work settings
    show_docs (bool): Enable API docs in web app by URL /docs.
        Default is True

    debug (bool): Debug mode. Use for FastAPI and for show model configs.
        Default is False

    throttling_max_requests (Optional[int]): Max count simultaneous requests to web app.
        If set None, throttling by requests is disable.
        Default is None

    throttling_max_request_len (Optional[int]): Max count objects to predict in single request.
        If set None, throttling by max items in request is disable.
        Default is None

    timeout_for_shutdown_daemon (float): Wait time for graceful shutdown web app.
        Default is 3.0

    uvicorn_kwargs (Dict): Uvicorn server kwargs arguments.
        Default is {}

    item_id_col_name (str): Column name for unique item_id.
        Use in batching and worker. It need for marks requests in worker by request id.
        Added this key for predicted data, but not thrown into model predict.
        Default is mlup.constants.ITEM_ID_COL_NAME - 'mlup_item_id'

    """
    # WebApp web outer interface settings
    host: str = '0.0.0.0'
    port: int = 8009
    web_app_version: str = '1.0.0.0'

    # Can use only single from two params
    column_validation: bool = False
    custom_column_pydantic_model: Optional[Type[PydanticBaseModel]] = None

    # WebApp architecture settings
    # WebApp work mode settings
    mode: WebAppArchitecture = WebAppArchitecture.directly_to_predict
    # Max queue size for waiting batching.
    max_queue_size: int = 100
    # Max time member predict result data. Need for clean if client not returned for predict result.
    ttl_predicted_data: int = 60
    # Max time wait results for clients, in single request.
    ttl_client_wait: float = 30.0
    # Min batch len for start predict.
    min_batch_len: int = 10
    # Max time for pending before run batching.
    batch_worker_timeout: float = 1.0
    # Added get-predict api method and add return predict_id to predict response.
    is_long_predict: bool = False

    # WebApp work settings
    # Enable API docs in web app
    show_docs: bool = True
    # Debug mode. Use for FastAPI and for show model configs
    debug: bool = False
    # Max count simultaneous requests to web app
    throttling_max_requests: Optional[int] = None
    # Max count objects to predict in single request
    throttling_max_request_len: Optional[int] = None
    # Wait time for graceful shutdown web app
    timeout_for_shutdown_daemon: float = 3.0
    # Uvicorn settings
    uvicorn_kwargs: Dict = field(default_factory=dict, repr=False)

    # Column name for unique item_id
    item_id_col_name: str = ITEM_ID_COL_NAME

    def wb_dict(self):
        res = {}
        for n, f in WebAppConfig.__dataclass_fields__.items():
            if f.repr is True:
                v = getattr(self, n)
                if isinstance(v, Enum):
                    v = v.value
                res[n] = v
        return res

    def wb_str(self, need_spaces: bool = False):
        res = []
        space = '    ' if need_spaces else ''
        for n, f in WebAppConfig.__dataclass_fields__.items():
            if f.repr is True:
                v = getattr(self, n)
                if isinstance(v, Enum):
                    v = v.value
                res.append(space + f'{n}={v}')
        return '\n'.join(res)


@dataclass(repr=True)
class MLupWebApp:
    """This is main UP web app class.
    Create object UP with your ML model, set your settings and run your web app.

    Args:
        ml (MLupModel): mlup model object.
        conf (WebAppConfig): Config for web app.
            If not set this class, config will create automatic with default params.

    """
    ml: MLupModel = field(repr=False)
    conf: WebAppConfig = field(default_factory=WebAppConfig, repr=False)

    # WebApp inner settings
    _fast_api: FastAPI = field(init=False, default=None, repr=False)
    _throttling_max_requests_current_count: int = field(init=False, default=0, repr=False)
    _predict_inner_pydantic_model: Type[MlModelPredictRequest] = field(init=False, repr=False)
    _daemon_thread: threading.Thread = field(init=False, default=None, repr=False)
    _uvicorn_server: uvicorn.Server = field(init=False, repr=False)
    _architecture_obj: BaseWebAppArchitecture = field(init=False, repr=False)

    def __del__(self):
        self.stop()

    def __getstate__(self):
        """Before pickle object"""
        logger.info(f'Running binarization {self}.')
        attributes = self.__dict__.copy()
        attributes.pop('_fast_api', None)
        attributes.pop('_predict_inner_pydantic_model', None)
        attributes.pop('_daemon_thread', None)
        attributes.pop('_uvicorn_server', None)
        attributes.pop('_architecture_obj', None)
        return attributes

    def __setstate__(self, state):
        """After unpickle object"""
        logger.info(f'Running an {self} load from binary data.')
        self.__dict__ = state
        self.load()

    @property
    def loaded(self) -> bool:
        return self._fast_api is not None

    @property
    def app(self) -> FastAPI:
        if not self.loaded:
            raise WebAppLoadError('web nor creating. Please call web.load().')
        return self._fast_api

    @asynccontextmanager
    async def _lifespan(self, app: FastAPI):
        # Startup web app code
        await self._architecture_obj.run()
        yield
        # Shutdown web app code
        await self._architecture_obj.stop()
        configure_logging_formatter('default')

    def _request_len_throttling(self, data_for_predict: Dict):
        x_for_predict = self.ml.get_X_from_predict_data(data_for_predict, remove=False)
        if x_for_predict and len(x_for_predict):
            raise ApiRequestError(
                'The query exceeded the limit on the number of rows for the predict. Please downsize your request.',
                status_code=429,
                type='throttling_error'
            )

    def _create_app(self):
        fast_api_kwargs = {}
        if self.conf.show_docs is False:
            fast_api_kwargs['docs_url'] = None
            fast_api_kwargs['redoc_url'] = None
        self._fast_api = FastAPI(
            debug=self.conf.debug,
            title=f"MLup web application with model: {self.ml.conf.name} v{self.ml.conf.version}.",
            description=f"Web application for use {self.ml.conf.name} v{self.ml.conf.version} in web.",
            version=self.conf.web_app_version,
            lifespan=self._lifespan,
            exception_handlers={
                RequestValidationError: api_exception_handler,
                ValidationError: api_exception_handler,
                ApiRequestError: api_request_error_handler,

                # Predict errors
                PredictError: predict_errors_handler,
                PredictWaitResultError: predict_errors_handler,
                PredictTransformDataError: predict_errors_handler,
                PredictValidationInnerDataError: predict_errors_handler,
            },
            responses={
                422: {
                    "description": "Error with validation input data",
                    "model": ApiErrorResponse,
                },
                429: {
                    "description": "Throttling input request error",
                    "model": ApiErrorResponse,
                },
                500: {
                    "description": "Error with predict process exception",
                    "model": ApiErrorResponse,
                }
            },
            **fast_api_kwargs,
        )

        # Set web api points
        self.app.add_api_route("/health", self.http_health, methods=["HEAD", "GET", "OPTIONS"])
        if self.conf.debug:
            self.app.add_api_route("/info", self.debug_info, methods=["GET"], name='info')
        else:
            self.app.add_api_route("/info", self.info, methods=["GET"], name='info')
        self.app.add_api_route("/predict", self.predict, methods=["POST"])

        self._architecture_obj = get_class_by_path(self.conf.mode)(
            fast_api=self._fast_api,
            ml=self.ml,
            item_id_col_name=self.conf.item_id_col_name,
            # Worker settings
            max_queue_size=self.conf.max_queue_size,
            ttl_predicted_data=self.conf.ttl_predicted_data,
            # Batching settings
            min_batch_len=self.conf.min_batch_len,
            batch_worker_timeout=self.conf.batch_worker_timeout,
            # Wait result from worker settings
            is_long_predict=self.conf.is_long_predict,
            ttl_client_wait=self.conf.ttl_client_wait,
        )
        self._predict_inner_pydantic_model = create_pydantic_predict_model(
            self.ml,
            self.conf.column_validation,
            custom_column_pydantic_model=self.conf.custom_column_pydantic_model,
        )

    def _run(self, in_thread: bool = False):
        if 'loop' not in self.conf.uvicorn_kwargs:
            self.conf.uvicorn_kwargs['loop'] = 'none' if in_thread else 'auto'
        if 'timeout_graceful_shutdown' not in self.conf.uvicorn_kwargs:
            self.conf.uvicorn_kwargs['timeout_graceful_shutdown'] = self.conf.timeout_for_shutdown_daemon
        if 'log_config' not in self.conf.uvicorn_kwargs:
            self.conf.uvicorn_kwargs['log_config'] = LOGGING_CONFIG
        configure_logging_formatter('web')

        logger.info(f'MLup application will be launched at: http://{self.conf.host}:{self.conf.port}')
        if self.conf.show_docs:
            logger.info(
                f"You can open your application's API documentation at http://{self.conf.host}:{self.conf.port}/docs"
            )

        self._uvicorn_server = uvicorn.Server(
            uvicorn.Config(self.app, **self.conf.uvicorn_kwargs),
        )
        self._uvicorn_server.run()

    def load_web_app_settings(self):
        """Load web app settings"""
        self.conf.uvicorn_kwargs['host'] = self.conf.host
        self.conf.uvicorn_kwargs['port'] = self.conf.port
        self.conf.uvicorn_kwargs['timeout_graceful_shutdown'] = int(self.conf.timeout_for_shutdown_daemon)

    def load(self):
        """Create and full load web app"""
        logger.info('Run load Web application')
        if not self.ml.loaded:
            raise WebAppLoadError('Model not loaded to memory. Analyze impossible. Please call ml.load().')

        logger.debug(f'Load Web application with settings:\n{self.conf.wb_str(need_spaces=True)}')
        if self.conf.throttling_max_requests is not None and self.conf.throttling_max_requests < 1:
            raise WebAppLoadError(
                'The param throttling_max_requests must be greater than 0. '
                f'Now it is {self.conf.throttling_max_requests}.'
            )
        if self.conf.throttling_max_request_len is not None and self.conf.throttling_max_request_len < 1:
            raise WebAppLoadError(
                'The param throttling_max_request_len must be greater than 0. '
                f'Now it is {self.conf.throttling_max_request_len}.'
            )

        if self.conf.column_validation and not self.ml.conf.columns:
            raise WebAppLoadError(
                'The param column_validation=True must use only, when there is ml.columns. '
                f'Now ml.columns is {self.ml.conf.columns}.'
            )
        if self.conf.column_validation and self.conf.custom_column_pydantic_model:
            raise WebAppLoadError(
                'Only one of the two parameters can be used: column_validation, custom_column_pydantic_model. '
                f'Now set column_validation={self.conf.column_validation}, '
                f'custom_column_pydantic_model={self.conf.custom_column_pydantic_model}.'
            )

        self.load_web_app_settings()
        self._create_app()
        self._architecture_obj.load()
        self.app.openapi = partial(openapi_schema, app=self.app, ml=self.ml)

    def stop(self, shutdown_timeout: Optional[float] = None):
        """
        Stop web app, if web app was runned in daemon mode.

        :param Optional[float] shutdown_timeout: Wait web_app gracefull shutdown.
            Use this param or self.conf.timeout_for_shutdown_daemon.

        """
        if self._daemon_thread and self._daemon_thread.is_alive():
            # Shutdown uvicorn not in main thread
            # https://stackoverflow.com/questions/58010119/are-there-any-better-ways-to-run-uvicorn-in-thread
            self._uvicorn_server.should_exit = True

            run_async(
                asyncio.wait_for,
                self._uvicorn_server.shutdown(),
                shutdown_timeout or self.conf.timeout_for_shutdown_daemon
            )
            self._daemon_thread.join(shutdown_timeout or self.conf.timeout_for_shutdown_daemon)

    def run(self, daemon: bool = False):
        """
        Run web app with ML model.

        :param bool daemon: Run web app with daemon mode and unblock terminal. Default is False.

        """
        if not self.ml.loaded:
            raise WebAppLoadError(
                'ML Model not loaded to memory. For run web app, please call ml.load(), or ml.load().'
            )

        if not self.loaded:
            raise WebAppLoadError('For run web app, please call web.load(), or web.load().')

        if daemon is not True:
            self._run(in_thread=False)
            return

        if self._daemon_thread and self._daemon_thread.is_alive():
            logger.error(f'WebApp is already running. Thread name {self._daemon_thread.name}')
            raise WebAppLoadError(f'WebApp is already running. Thread name {self._daemon_thread.name}')

        self._daemon_thread = threading.Thread(
            target=self._run,
            kwargs={'in_thread': True},
            daemon=False,
            name='MLupWebAppDaemonThread'
        )
        self._daemon_thread.start()
        logger.info('Waiting start uvicorn proc with web app 30.0 seconds.')
        time.sleep(0.1)
        time_run = time.monotonic()
        while not self._uvicorn_server and time.monotonic() - time_run < 30.0:
            time.sleep(0.1)
        while self._uvicorn_server.started is False and time.monotonic() - time_run < 30.0:
            time.sleep(0.1)

    async def http_health(self):
        return {'status': 200}

    async def debug_info(self):
        info = await self.info()
        info.update({
            "model_config": self.ml.conf.ml_dict(),
            "web_app_config": self.conf.wb_dict(),
        })
        return info

    async def info(self):
        return {
            "model_info": {
                "name": self.ml.conf.name,
                "version": self.ml.conf.version,
                "type": self.ml.conf.type,
                "columns": self.ml.conf.columns,
            },
            "web_app_info": {
                "version": self.conf.web_app_version,
            },
        }

    @_set_predict_id_to_response_headers
    @_requests_throttling
    async def predict(self, request: FastAPIRequest, response: FastAPIResponse):
        predict_id = response.headers[PREDICT_ID_HEADER]
        # Validation
        try:
            predict_request_body = self._predict_inner_pydantic_model(**(await request.json()))
        except json.JSONDecodeError as e:
            raise PredictValidationInnerDataError(msg=f'Invalid json data: {e}', predict_id=predict_id)
        data_for_predict = predict_request_body.dict()

        # Throttling
        if self.conf.throttling_max_request_len:
            self._request_len_throttling(data_for_predict)

        # Predict in web app architecture object
        predict_result = await self._architecture_obj.predict(data_for_predict, predict_id=predict_id)

        # Result
        return {"predict_result": predict_result}

mlup/web/app.py
nxexox/pymlup
import abc
from typing import Dict, Any

from mlup.ml.model import MLupModel
from mlup.constants import WebAppArchitecture


class BaseWebAppArchitecture(metaclass=abc.ABCMeta):
    ml: MLupModel
    type: WebAppArchitecture

    @abc.abstractmethod
    def load(self):
        pass

    @abc.abstractmethod
    async def run(self):
        pass

    @property
    @abc.abstractmethod
    def is_running(self) -> bool:
        pass

    @abc.abstractmethod
    async def stop(self):
        pass

    @abc.abstractmethod
    async def predict(self, data_for_predict: Dict, predict_id: str) -> Any:
        pass

mlup/web/architecture/base.py
nxexox/pymlup
import asyncio
import logging
import time
from asyncio import Queue, Task
from typing import Any, Dict, List, Optional

from fastapi import FastAPI

from mlup.errors import PredictWaitResultError, WebAppLoadError, PredictError
from mlup.ml.model import MLupModel
from mlup.utils.collections import TTLOrderedDict
from mlup.web.architecture.base import BaseWebAppArchitecture
from mlup.constants import WebAppArchitecture


logger = logging.getLogger('mlup')


class BatchingSingleProcessArchitecture(BaseWebAppArchitecture):
    fast_api: FastAPI
    ml: MLupModel
    item_id_col_name: str

    # Min batch len for start predict.
    min_batch_len: int
    # Max time for pending before run batching.
    batch_worker_timeout: float
    # Max queue size for waiting batching.
    max_queue_size: int
    # Max time member predict result data. Need for clean if client not returned for predict result.
    ttl_predicted_data: int
    # Max time wait results for clients, in single request.
    ttl_client_wait: float
    # Added get-predict api method and add return predict_id to predict response.
    is_long_predict: bool

    model_data_col_name: str = 'data_for_predict'
    results_storage: Dict = None
    queries_queue: Queue = None
    worker_process: Task = None
    batch_queue: List = None
    batch_predict_ids: List = None
    type: WebAppArchitecture = WebAppArchitecture.batching

    def __init__(self, **configs):
        self.fast_api = configs.pop('fast_api')
        self.ml = configs.pop('ml')
        self.item_id_col_name = configs.pop('item_id_col_name')
        self.min_batch_len = configs.pop('min_batch_len')
        self.batch_worker_timeout = configs.pop('batch_worker_timeout')
        self.max_queue_size = configs.pop('max_queue_size')
        self.ttl_predicted_data = configs.pop('ttl_predicted_data')
        self.ttl_client_wait = configs.pop('ttl_client_wait')
        self.is_long_predict = configs.pop('is_long_predict')

        if self.batch_worker_timeout <= 0:
            raise ValueError(
                f'The param batch_worker_timeout parameter must be greater than 0. '
                f'Now it is {self.batch_worker_timeout}'
            )
        if self.ttl_predicted_data <= 0:
            raise ValueError(
                f'The param ttl_predicted_data parameter must be greater than 0. '
                f'Now it is {self.ttl_predicted_data}'
            )

        self._running = False
        self.batch_queue = []
        self.batch_predict_ids = []

        self.extra = configs

    async def _get_result_from_storage(self, predict_id: str, sleep_time: float = 0.1):
        _start = time.monotonic()
        while True:
            if (time.monotonic() - _start) >= self.ttl_client_wait:
                raise PredictWaitResultError(
                    'Response timed out. Repeat request please.',
                    predict_id=predict_id,
                )
            if predict_id in self.results_storage.keys():
                predict_task_result, error = self.results_storage.get(predict_id)
                if error is not None:
                    raise error
                return predict_task_result
            await asyncio.sleep(sleep_time)

    def _save_predicted_data_from_batch(self, predicted_data: List, error: Optional[Exception]):
        current_predict_id_index = 0
        count_saved = 0
        local_result = []
        for i, item in enumerate(predicted_data):
            local_result.append(item)
            predict_id, predict_id_current_max = self.batch_predict_ids[current_predict_id_index]
            if i == count_saved + predict_id_current_max - 1:
                current_predict_id_index += 1
                self.results_storage[predict_id] = (local_result, error)
                count_saved += len(local_result)
                local_result = []

    async def _predict(self):
        logger.debug('Run predict for batch')
        predicted_data = self.batch_queue
        error = None
        try:
            predicted_data = await self.ml.predict(
                **{self.ml.x_column_name: predicted_data}
            )
        except Exception as e:
            error = e

        self._save_predicted_data_from_batch(predicted_data, error)
        self.batch_queue.clear()
        self.batch_predict_ids.clear()
        logger.debug('End predict for batch')

    async def _start_worker(self, sleep_time: float = 0.1):
        self._running = True
        logger.info('Start checking the batch queue...')
        while self._running:
            await asyncio.sleep(min(sleep_time, self.batch_worker_timeout))
            if not self.queries_queue.empty():
                _count_queue = worked_time = 0
                started_at = time.monotonic()
                while not self._is_batch_done(worked_time):
                    if self.queries_queue.qsize() > 0:
                        queue_data = await self.queries_queue.get()
                        _count_queue += 1
                        predict_id = queue_data[self.item_id_col_name]
                        data_for_predict = queue_data[self.model_data_col_name]
                        self.batch_queue.extend(data_for_predict)
                        self.batch_predict_ids.append((predict_id, len(data_for_predict)))
                    else:
                        await asyncio.sleep(min(sleep_time, self.batch_worker_timeout))
                    worked_time = time.monotonic() - started_at
                logger.info(f'Batch handling started after {worked_time:.2f} seconds for {_count_queue} requests')
                logger.info(f'Batch length {len(self.batch_queue)}.')
                logger.info(f'Predict ids in batch: {"|".join(next(zip(*self.batch_predict_ids)))}')
                await self._predict()
        logger.info('Stop checking the batch queue...')

    def _is_batch_done(self, worked_time: float) -> bool:
        return len(self.batch_queue) >= self.min_batch_len or worked_time >= self.batch_worker_timeout

    def load(self):
        logger.debug(
            f'Load {WebAppArchitecture.batching.value} with params:\n'
            f'    item_id_col_name={self.item_id_col_name}\n'
            f'    min_batch_len={self.min_batch_len}\n'
            f'    max_queue_size={self.max_queue_size}\n'
            f'    batch_worker_timeout={self.batch_worker_timeout}\n'
            f'    ttl_predicted_data={self.ttl_predicted_data}\n'
            f'    ttl_client_wait={self.ttl_client_wait}\n'
            f'    is_long_predict={self.is_long_predict}'
        )
        self.results_storage = TTLOrderedDict(self.ttl_predicted_data)
        self.queries_queue = Queue(maxsize=self.max_queue_size)

        if self.is_long_predict:
            self.fast_api.add_api_route("/get-predict/{predict_id}", self.get_predict_result, methods=["GET"])

    async def run(self):
        if self.is_running:
            raise WebAppLoadError('Worker is already running')
        logger.info('Run model in batching worker')
        if self.results_storage is None or self.queries_queue is None:
            raise WebAppLoadError('Not called .load() in web.load()')
        self.worker_process = Task(self._start_worker())

    @property
    def is_running(self) -> bool:
        return self._running

    async def stop(self):
        if not self.is_running:
            raise WebAppLoadError('Worker not started')
        self._running = False
        if not self.worker_process.cancelled():
            await self.worker_process
            self.worker_process.cancel()
        logger.info('Batching worker stopped')

    async def get_predict_result(self, predict_id: str):
        return await self._get_result_from_storage(predict_id)

    async def predict(self, data_for_predict: Dict, predict_id: str) -> Any:
        if not self.is_running:
            raise PredictError('Worker not started. Please call web.run()')

        x_for_predict = self.ml.get_X_from_predict_data(data_for_predict)

        data_for_queue = {
            self.item_id_col_name: predict_id,
            self.model_data_col_name: x_for_predict,
        }

        logger.info(f'New request {predict_id} put to queue.')
        try:
            self.queries_queue.put_nowait(data_for_queue)
        except asyncio.queues.QueueFull:
            raise PredictError('Queue is full. Please try later.')
        logger.debug(f'Request {predict_id} wait predicted')

        if self.is_long_predict:
            return {"predict_id": predict_id}

        return await self._get_result_from_storage(predict_id)

mlup/web/architecture/batching.py
nxexox/pymlup
import logging
from typing import Dict, Any

from mlup.ml.model import MLupModel
from mlup.web.architecture.base import BaseWebAppArchitecture
from mlup.constants import WebAppArchitecture

logger = logging.getLogger('mlup')


class DirectlyToPredictArchitecture(BaseWebAppArchitecture):
    ml: MLupModel
    type: WebAppArchitecture = WebAppArchitecture.directly_to_predict

    def __init__(self, **configs):
        self.ml = configs.pop('ml')
        self.extra = configs

    def load(self):
        logger.info(f'Load {self.type}')

    async def run(self):
        logger.debug('Run model directly to predict')

    @property
    def is_running(self) -> bool:
        return True

    async def stop(self):
        logger.debug('Model directly stopped')

    async def predict(self, data_for_predict: Dict, predict_id: str) -> Any:
        logger.info(f'New request {predict_id}.')
        logger.debug(f'Request {predict_id} wait predicted')
        result = await self.ml.predict(**data_for_predict)
        logger.info(f'End predict for {predict_id}')
        return result

mlup/web/architecture/directly_to_predict.py
nxexox/pymlup
import asyncio
import logging
import time
from asyncio import Queue, Task
from typing import Dict, Any

from fastapi import FastAPI

from mlup.errors import WebAppLoadError, PredictError, PredictWaitResultError
from mlup.ml.model import MLupModel
from mlup.utils.collections import TTLOrderedDict
from mlup.utils.loop import create_async_task
from mlup.web.architecture.base import BaseWebAppArchitecture
from mlup.constants import WebAppArchitecture

logger = logging.getLogger('mlup')


class WorkerAndQueueArchitecture(BaseWebAppArchitecture):
    fast_api: FastAPI
    ml: MLupModel
    item_id_col_name: str

    # Worker settings
    # Max queue size for waiting predict.
    max_queue_size: int
    # Max time member predict result data. Need for clean if client not returned for predict result.
    ttl_predicted_data: int
    # Added get-predict api method and add return predict_id to predict response.
    is_long_predict: bool
    # Max time wait results for clients, in single request.
    ttl_client_wait: float

    model_data_col_name: str = 'data_for_predict'
    results_storage: Dict = None
    queries_queue: Queue = None
    worker_process: Task = None
    type: WebAppArchitecture = WebAppArchitecture.worker_and_queue

    def __init__(self, **configs):
        self.fast_api = configs.pop('fast_api')
        self.ml = configs.pop('ml')
        self.item_id_col_name = configs.pop('item_id_col_name')
        self.max_queue_size = configs.pop('max_queue_size')
        self.ttl_predicted_data = configs.pop('ttl_predicted_data')
        self.is_long_predict = configs.pop('is_long_predict')
        self.ttl_client_wait = configs.pop('ttl_client_wait')

        if self.ttl_predicted_data <= 0:
            raise ValueError(
                f'The param ttl_predicted_data parameter must be greater than 0. '
                f'Now it is {self.ttl_predicted_data}'
            )

        self._running = False

        self.extra = configs

    async def _get_result_from_storage(self, predict_id: str, sleep_time: float = 0.1):
        _start = time.monotonic()
        while True:
            if (time.monotonic() - _start) >= self.ttl_client_wait:
                raise PredictWaitResultError(
                    'Response timed out. Repeat request please.',
                    predict_id=predict_id,
                )
            if predict_id in self.results_storage.keys():
                predict_task_result, error = self.results_storage.get(predict_id)
                if error is not None:
                    raise error
                return predict_task_result
            await asyncio.sleep(sleep_time)

    async def _predict(self, predict_id: str, data_for_predict: Dict) -> Any:
        predicted_data = None
        error = None
        try:
            predicted_data = await self.ml.predict(**data_for_predict)
        except Exception as e:
            error = e

        self.results_storage[predict_id] = (predicted_data, error)

    async def _start_worker(self, sleep_time: float = 0.1):
        self._running = True
        logger.info('Start checking the queue...')

        while self._running:
            await asyncio.sleep(sleep_time)
            if self.queries_queue.empty():
                continue

            queue_data = await self.queries_queue.get()
            predict_id, data_for_predict = queue_data[self.item_id_col_name], queue_data[self.model_data_col_name]
            logger.info(f'Run predict for {predict_id}')
            await self._predict(predict_id, data_for_predict)
            logger.info(f'End predict for {predict_id}')
        logger.info('Stop checking the queue...')

    async def run(self):
        if self.is_running:
            raise WebAppLoadError('Worker is already running')
        logger.info('Run model in worker')
        if self.results_storage is None or self.queries_queue is None:
            raise WebAppLoadError('Not called .load() in web.load()')
        self.worker_process = create_async_task(self._start_worker())

    @property
    def is_running(self) -> bool:
        return self._running

    async def stop(self):
        if not self.is_running:
            raise WebAppLoadError('Worker not started')
        self._running = False
        if not self.worker_process.cancelled():
            await self.worker_process
            self.worker_process.cancel()
        logger.info('Worker stopped')

    def load(self):
        logger.debug(
            f'Load {WebAppArchitecture.worker_and_queue.value} with params:\n'
            f'    item_id_col_name={self.item_id_col_name}\n'
            f'    max_queue_size={self.max_queue_size}\n'
            f'    ttl_predicted_data={self.ttl_predicted_data}\n'
            f'    is_long_predict={self.is_long_predict}\n'
            f'    ttl_client_wait={self.ttl_client_wait}'
        )
        self.results_storage = TTLOrderedDict(self.ttl_predicted_data)
        self.queries_queue = Queue(maxsize=self.max_queue_size)

        if self.is_long_predict:
            self.fast_api.add_api_route("/get-predict/{predict_id}", self.get_predict_result, methods=["GET"])

    async def get_predict_result(self, predict_id: str):
        return await self._get_result_from_storage(predict_id)

    async def predict(self, data_for_predict: Dict, predict_id: str) -> Any:
        if not self.is_running:
            raise PredictError('Worker not started. Please call web.run()')

        data_for_queue = {
            self.item_id_col_name: predict_id,
            self.model_data_col_name: data_for_predict
        }

        logger.info(f'New request {predict_id} put to queue.')
        try:
            self.queries_queue.put_nowait(data_for_queue)
        except asyncio.queues.QueueFull:
            raise PredictError('Queue is full. Please try later.')
        logger.debug(f'Request {predict_id} wait predicted')

        if self.is_long_predict:
            return {"predict_id": predict_id}

        return await self._get_result_from_storage(predict_id)

mlup/web/architecture/worker_and_queue.py
nxexox/pymlup
import json
import logging
import os
import shutil
import socket
import sys
from dataclasses import dataclass, field
import pickle
import time
from itertools import cycle
from pathlib import Path
from typing import List, Type, Any, Dict

import httpx
import joblib
import numpy as np
import pytest
import yaml

import mlup
from mlup.constants import DEFAULT_X_ARG_NAME, StorageType, ModelDataTransformerType
from mlup.web.app import MLupWebApp


logger = logging.getLogger('mlup.test')


_test_json_config = {
    "ml": {
        "binarization_type": "mlup.ml.binarization.pickle.PickleBinarizer",
        "data_transformer_for_predicted": "mlup.ml.data_transformers.numpy_data_transformer.NumpyDataTransformer",
        "data_transformer_for_predict": "mlup.ml.data_transformers.pandas_data_transformer.PandasDataFrameTransformer",
        "use_thread_loop": True,
        "columns": [
            {"name": "MinTemp", "type": "float", "required": False, "default": 1.4},
            {"name": "MaxTemp", "type": "float", "required": False},
            {"name": "Humidity9am", "type": "float", "required": False},
            {"name": "Humidity3pm", "type": "float", "required": False},
            {"name": "Pressure9am", "type": "float", "required": False},
            {"name": "Pressure3pm", "type": "float", "required": False},
            {"name": "Temp9am", "type": "float", "required": False},
            {"name": "Temp3pm", "type": "float", "required": False}
        ],
        "storage_type": "mlup.ml.storage.local_disk.DiskStorage",
        "storage_kwargs": {
            "path_to_files": "test/path/to/file.pckl",
        },
        "auto_detect_predict_params": True,
        "max_thread_loop_workers": None,
        "type": "sklearn",
        "version": "1.2.3.4",
        "predict_method_name": "predict",
        "name": "Test"
    },
    "web": {
        "host": "0.0.0.0",
        "column_validation": False,
        "mode": "mlup.web.architecture.directly_to_predict.DirectlyToPredictArchitecture",
        "web_app_version": "1.0.0.0",
        "min_batch_len": 10,
        "port": 8009,
        "ttl_predicted_data": 60,
        "throttling_max_requests": None,
        "item_id_col_name": "mlup_item_id",
        "is_long_predict": False,
        "batch_worker_timeout": 1,
        "max_queue_size": 100,
        "throttling_max_request_len": None,
        "timeout_for_shutdown_daemon": 3,
        "debug": False,
        "show_docs": True,
        "ttl_client_wait": 30.0,
        "uvicorn_kwargs": {},
    }
}


@pytest.fixture(scope="session")
def test_dict_config():
    return _test_json_config


@pytest.fixture(scope="session")
def test_yaml_config(tmp_path_factory):
    f_path = tmp_path_factory.getbasetemp()
    config_name = 'yaml_test_config.yaml'
    logger.info(f'Create {f_path}/{config_name} fixture.')
    with open(f_path / config_name, 'w') as f:
        yaml.dump(_test_json_config, f)
    return f_path / config_name


@pytest.fixture(scope="session")
def test_json_config(tmp_path_factory):
    f_path = tmp_path_factory.getbasetemp()
    config_name = 'yaml_test_config.json'
    logger.info(f'Create {f_path}/{config_name} fixture.')
    with open(f_path / config_name, 'w') as f:
        json.dump(_test_json_config, f)
    return f_path / config_name


@dataclass
class TestMlupWebClient:
    """
    This client maked because FastApi client fastapi.testclient.TestClient
    сrashed when used multiple times with multiple applications.

    This class starts web with run(daemon=True).

    """
    app: MLupWebApp
    wait_running: float = 1
    wait_after_shutdown: float = 3

    def __enter__(self):
        self.app.conf.host = 'localhost'
        self.app.conf.port = self.get_free_port()
        self.app.run(daemon=True)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        try:
            self.app.stop(shutdown_timeout=self.app.conf.timeout_for_shutdown_daemon)
        except Exception as e:
            logger.error(f'It is OK. Shutdown web error, before unicorn.shutdown is stuck: {e}')

    @classmethod
    def get_free_port(cls) -> int:
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
            for port in cycle(range(8009, 8015)):
                r = s.connect_ex(('localhost', port))
                if r != 0:
                    return port

    async def get(self, url, params=None) -> httpx.Response:
        async with httpx.AsyncClient() as client:
            return await client.get(f'http://localhost:{self.app.conf.port}{url}', params=params)

    async def post(self, url, params=None, json=None) -> httpx.Response:
        async with httpx.AsyncClient() as client:
            return await client.post(f'http://localhost:{self.app.conf.port}{url}', params=params, json=json)


@pytest.fixture
def web_app_test_client():
    return TestMlupWebClient


class TestModel:
    def run_predict(self, X: List, test_param: bool = False, *custom_args, **custom_kwargs):
        pass

    def predict(self, X: List, test_param: bool = False, *custom_args, **custom_kwargs):
        return self.run_predict(X, test_param, *custom_args, **custom_kwargs)

    def second_predict(self, X: List, test_param: bool = False, *custom_args, **custom_kwargs):
        return self.run_predict(X, test_param, *custom_args, **custom_kwargs)

    def predict_with_x_name_y(self, Y: List, test_param: bool = False, *custom_args, **custom_kwargs):
        return self.run_predict(Y, test_param, *custom_args, **custom_kwargs)


class PrintModel(TestModel):
    def run_predict(self, X: List, test_param: bool = False, *custom_args, **custom_kwargs):
        print(
            f'Call PrintModel.run_predict('
            f'X={X}, test_param={test_param}, *custom_args={custom_args}, custom_kwargs={custom_kwargs}'
            f')'
        )
        return X


@dataclass
class PrintSleepModel(TestModel):
    sleep: float = 1

    def run_predict(self, X: List, test_param: bool = False, *custom_args, **custom_kwargs):
        time.sleep(self.sleep)
        print(
            f'Call PrintSleepModel.run_predict('
            f'X={X}, test_param={test_param}, *custom_args={custom_args}, custom_kwargs={custom_kwargs}'
            f')'
        )
        return X


class ListToNumpyArrayModel(TestModel):
    def run_predict(self, X: List, test_param: bool = False, *custom_args, **custom_kwargs):
        print(
            f'Call ListToNumpyArrayModel.run_predict('
            f'X={X}, test_param={test_param}, *custom_args={custom_args}, custom_kwargs={custom_kwargs}'
            f')'
        )
        return np.array(X)


class RaiseExceptionModel(TestModel):
    exc = ValueError('Test message')

    def run_predict(self, X: List, test_param: bool = False, *custom_args, **custom_kwargs):
        raise self.exc


class ModelWithX:
    def predict(self, X: List):
        return X


@pytest.fixture(scope="function")
def print_model() -> PrintModel:
    return PrintModel()


@pytest.fixture(scope="function")
def print_sleep_model() -> PrintSleepModel:
    return PrintSleepModel()


@pytest.fixture(scope="function")
def model_with_x() -> ModelWithX:
    return ModelWithX()


@pytest.fixture(scope="session")
def model_with_x_class() -> Type[ModelWithX]:
    return ModelWithX


@pytest.fixture(scope="function")
def raise_exception_model() -> RaiseExceptionModel:
    return RaiseExceptionModel()


@pytest.fixture(scope="function")
def list_to_numpy_array_model() -> ListToNumpyArrayModel:
    return ListToNumpyArrayModel()


@pytest.fixture(scope="session")
def pickle_print_model(tmp_path_factory):
    model = PrintModel()
    f_path = tmp_path_factory.getbasetemp()
    model_name = 'pickle_print_model.pckl'
    logger.info(f'Create {f_path}/{model_name} fixture.')
    with open(f_path / model_name, 'wb') as f:
        pickle.dump(model, f)
    return f_path / model_name


@pytest.fixture(scope="session")
def pickle_print_sleep_model(tmp_path_factory):
    model = PrintSleepModel(sleep=1)
    f_path = tmp_path_factory.getbasetemp()
    model_name = 'pickle_print_sleep_model.pckl'
    logger.info(f'Create {f_path}/{model_name} fixture.')
    with open(f_path / model_name, 'wb') as f:
        pickle.dump(model, f)
    return f_path / model_name


@pytest.fixture(scope="session")
def joblib_print_model(tmp_path_factory):
    model = PrintModel()
    f_path = tmp_path_factory.getbasetemp()
    model_name = 'joblib_print_model.joblib'
    logger.info(f'Create {f_path}/{model_name} fixture.')
    with open(f_path / model_name, 'wb') as f:
        joblib.dump(model, f)
    return f_path / model_name


@pytest.fixture(scope="session")
def joblib_print_sleep_model(tmp_path_factory):
    model = PrintSleepModel(sleep=1)
    f_path = tmp_path_factory.getbasetemp()
    model_name = 'joblib_print_sleep_model.joblib'
    logger.info(f'Create {f_path}/{model_name} fixture.')
    with open(f_path / model_name, 'wb') as f:
        joblib.dump(model, f)
    return f_path / model_name


@pytest.fixture(scope="session")
def pickle_not_exists_file(tmp_path_factory):
    return tmp_path_factory.getbasetemp() / 'not_exists_file.pckl'


@pytest.fixture(scope="session")
def pickle_not_exists_folder(tmp_path_factory):
    return tmp_path_factory.getbasetemp() / 'not_exists_folder'


@pytest.fixture(scope="session")
def root_dir(request):
    return request.config.rootdir


@pytest.fixture(scope="session")
def models_datadir(root_dir):
    return root_dir / 'mldata' / 'models'


@pytest.fixture(scope="session")
def tests_jupyter_notebooks_datadir():
    return 'tests/integration_tests/notebooks'


@dataclass
class ModelAndPath:
    path: str
    model: Any
    test_data_raw: Dict = field(default_factory=lambda: {
        'MinTemp': 1.0,
        'MaxTemp': 2.0,
        'Humidity9am': 3.0,
        'Humidity3pm': 4.0,
        'Pressure9am': 5.0,
        'Pressure3pm': 6.0,
        'Temp9am': 7.0,
        'Temp3pm': 8.0,
    })
    test_model_response_raw: Any = 'No'
    predict_method_name: str = '__call__'
    file_mask: str = r'(\w.-_)*.pckl'
    is_many_files: bool = False
    x_arg_name: str = DEFAULT_X_ARG_NAME


@pytest.fixture(scope="session")
def scikit_learn_binary_cls_model(models_datadir):
    try:
        import sklearn
        with open(models_datadir / 'scikit-learn-binary_cls_model.pckl', 'rb') as f:
            model = pickle.load(f)
        return ModelAndPath(
            models_datadir / 'scikit-learn-binary_cls_model.pckl',
            model,
            test_model_response_raw='No',
            predict_method_name='predict',
            x_arg_name='X',
        )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def pickle_scikit_learn_model_config_yaml(tmp_path_factory, scikit_learn_binary_cls_model):
    f_path = tmp_path_factory.getbasetemp()
    config_name = 'pickle_scikit_learn_model_conf.yaml'
    logger.info(f'Create {f_path}/{config_name} fixture.')
    up = mlup.UP(
        conf=mlup.Config(
            storage_type=StorageType.disk,
            storage_kwargs={'path_to_files': str(scikit_learn_binary_cls_model.path)},
            data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
        )
    )
    up.to_yaml(f_path / config_name)
    return f_path / config_name


@pytest.fixture(scope="session")
def scikit_learn_binary_cls_model_onnx(models_datadir):
    try:
        import onnxruntime as onnxrt
        from mlup.ml.binarization.onnx import _InferenceSessionWithPredict
        model = _InferenceSessionWithPredict(str(models_datadir / 'scikit-learn-binary_cls_model.onnx'))
        return ModelAndPath(
            models_datadir / 'scikit-learn-binary_cls_model.onnx',
            model,
            test_model_response_raw='No',
            predict_method_name='predict',
            x_arg_name='input_data',
        )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def lightgbm_binary_cls_model(models_datadir):
    try:
        with open(models_datadir / 'lightgbm-binary_cls_model.pckl', 'rb') as f:
            return ModelAndPath(
                models_datadir / 'lightgbm-binary_cls_model.pckl',
                pickle.load(f),
                test_model_response_raw=0.19782479223155408,
                x_arg_name='data',
            )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def lightgbm_binary_cls_model_txt(models_datadir):
    try:
        import lightgbm as lgb
        return ModelAndPath(
            models_datadir / 'lightgbm-binary_cls_model.txt',
            lgb.Booster(model_file=models_datadir / 'lightgbm-binary_cls_model.txt'),
            test_model_response_raw=0.19782479223155408,
            x_arg_name='data',
            file_mask=r'(\w.-_)*.txt',
        )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def tensorflow_binary_cls_model(models_datadir):
    try:
        import tensorflow
        model_name = 'tensorflow-binary_cls_model.pckl'
        model_result = 0.5144115686416626
        # For python 3.7
        if sys.version_info.minor == 7:
            model_name = 'tensorflow-binary_cls_model37.pckl'
            model_result = 0.6405760645866394

        with open(models_datadir / model_name, 'rb') as f:
            return ModelAndPath(
                models_datadir / model_name,
                pickle.load(f),
                test_model_response_raw=model_result,
            )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def tensorflow_binary_cls_model_keras(models_datadir, tmp_path_factory):
    try:
        import tensorflow
        model_name = 'tensorflow-binary_cls_model.keras'
        model_result = 0.5144115686416626
        # For python 3.7
        if sys.version_info.minor == 7:
            model_name = 'tensorflow-binary_cls_model37.keras'
            model_result = 0.6405760645866394

        path_to_model = models_datadir / model_name
        model = tensorflow.keras.models.load_model(str(path_to_model), compile=False)
        model.compile()

        return ModelAndPath(
            path_to_model,
            model=model,
            test_model_response_raw=model_result,
            file_mask=Path(path_to_model).name,
        )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def tensorflow_binary_cls_model_h5(models_datadir):
    try:
        import tensorflow
        model_name = 'tensorflow-binary_cls_model.h5'
        model_result = 0.5144115686416626
        # For python 3.7
        if sys.version_info.minor == 7:
            model_name = 'tensorflow-binary_cls_model37.h5'
            model_result = 0.6405760645866394

        path_to_model = models_datadir / model_name
        model = tensorflow.keras.models.load_model(path_to_model, compile=False)
        model.compile()

        return ModelAndPath(
            path_to_model,
            model=model,
            test_model_response_raw=model_result,
            file_mask=r'(\w.-_)*.h5',
        )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def tensorflow_binary_cls_model_zip(models_datadir):
    try:
        import tensorflow
        # Only python3.8+
        return ModelAndPath(
            models_datadir,
            model=tensorflow.saved_model.load(models_datadir / 'tensorflow-binary_cls_model.savedmodel'),
            test_model_response_raw=0.5144115686416626,
            file_mask=r'tensorflow-binary_cls_model.savedmodel',
            is_many_files=True,
            predict_method_name='serve',
            x_arg_name='args_0'
        )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def pytorch_binary_cls_model(models_datadir):
    try:
        import torch
        with open(models_datadir / 'pytorch-binary_cls_model.pckl', 'rb') as f:
            return ModelAndPath(
                models_datadir / 'pytorch-binary_cls_model.pckl',
                pickle.load(f),
                test_model_response_raw=0.4163038432598114,
            )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def pytorch_binary_cls_model_onnx(models_datadir):
    try:
        import onnxruntime as onnxrt
        from mlup.ml.binarization.onnx import _InferenceSessionWithPredict
        model = _InferenceSessionWithPredict(str(models_datadir / 'pytorch-binary_cls_model.onnx'))
        return ModelAndPath(
            models_datadir / 'pytorch-binary_cls_model.onnx',
            model,
            test_model_response_raw=0.4163038432598114,
            predict_method_name='predict',
            x_arg_name='input',
        )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def pytorch_binary_cls_model_pth(models_datadir):
    try:
        import torch
        with open(models_datadir / 'pytorch-binary_cls_model.pth', 'rb') as f:
            model = torch.load(f)
            model.eval()
        return ModelAndPath(
            models_datadir / 'pytorch-binary_cls_model.pth',
            model,
            file_mask=r'(\w.-_)*.pth',
            test_model_response_raw=0.4163038432598114,
        )
    except ImportError:
        return None


@pytest.fixture(scope="session")
def pytorch_binary_cls_model_jit(models_datadir):
    try:
        import torch
        import io
        with open(models_datadir / 'pytorch-binary_cls_model-jit.pth', 'rb') as f:
            model = torch.jit.load(io.BytesIO(f.read()))
        model.eval()
        return ModelAndPath(
            models_datadir / 'pytorch-binary_cls_model-jit.pth',
            model,
            file_mask=r'(\w.-_)*.pth',
            test_model_response_raw=0.4163038432598114,
        )
    except ImportError:
        return None

tests/conftest.py
nxexox/pymlup
import json
import logging
import subprocess
import uuid
from dataclasses import dataclass, field
import datetime
from time import sleep
from typing import Dict, Optional, List

import pytest
import requests
from websocket import create_connection


logger = logging.getLogger('mlup.test')


class JupyterCellCallError(Exception):
    def __init__(self, content, msg):
        self.content = content
        self.msg = msg

    def __str__(self):
        return f'Content={self.content}, msg={self.msg}'


@dataclass
class JupyterNotebookTestServer:
    ip: str = '0.0.0.0'
    port: int = 8888
    protocol: str = 'http'
    token = 'mytoken123456'
    notebooks_folder: str = ''

    _server_start_command_tmp: str = "jupyter notebook --ip {ip} --port {port} --NotebookApp.token='{token}' " \
                                     "--no-browse {notebooks_folder}"
    _server_stop_command_tmp: str = "jupyter notebook stop {port}"
    _start_proc: subprocess.Popen = field(init=False, repr=False)
    _stop_proc: subprocess.Popen = field(init=False, repr=False)

    @property
    def headers(self) -> Dict:
        return {"Authorization": f"Token {self.token}"}

    @property
    def server_host(self) -> str:
        return f'{self.protocol}://{self.ip}:{self.port}'

    def start_server(self):
        self._start_proc = subprocess.Popen(
            self._server_start_command_tmp.format(
                ip=self.ip, port=self.port, token=self.token, notebooks_folder=self.notebooks_folder,
            ).split()
        )
        logger.info('Waiting start jupyter notebooks server 5 seconds')
        sleep(5)

    def copy_notebook_to_work_folder(self, notebook_name: str):
        pass

    def stop_server(self) -> Optional[str]:
        if self._start_proc is None:
            return None
        self._stop_proc = subprocess.Popen(
            self._server_stop_command_tmp.format(port=self.port).split()
        )
        output, error = self._stop_proc.communicate()
        if error:
            raise ValueError(str(error))
        self._stop_proc = None
        self._stop_proc = None
        return output

    def send_execute_request(self, code):
        msg_type = "execute_request"
        content = {"code": code, "silent": False}
        hdr = {
            "msg_id": uuid.uuid1().hex,
            "username": "test",
            "session": uuid.uuid1().hex,
            "data": datetime.datetime.now().isoformat(),
            "msg_type": msg_type,
            "version": "5.0",
        }
        msg = {"header": hdr, "parent_header": hdr, "metadata": {}, "content": content}
        return msg

    def parse_notebook_results(self, results: List[Dict]):
        res = []
        for r in results:
            logger.info(f'Result call cell {r}')
            _d = r['data']
            if 'text/plain' in _d:
                res.append(_d['text/plain'])
            else:
                res.append(_d)
        return res

    def run_notebook(self, notebook_path: str):
        url = f"{self.server_host}/api/kernels"
        with requests.post(url, headers=self.headers) as response:
            kernel = json.loads(response.text)

        # Load the notebook and get the code of each cell
        url = f"{self.server_host}/api/contents/{notebook_path}"
        with requests.get(url, headers=self.headers) as response:
            file = json.loads(response.text)

        # filter out non-code cells like markdown
        code = [
            c["source"]
            for c in file["content"]["cells"]
            if len(c["source"]) > 0 and c["cell_type"] == "code"
        ]

        # Execution request/reply is done on websockets channels
        ws = create_connection(
            f"{'ws' if self.protocol == 'http' else 'wss'}://{self.ip}:{self.port}/api/kernels/{kernel['id']}/channels",
            header=self.headers,
        )

        for c in code:
            ws.send(json.dumps(self.send_execute_request(c)))

        code_blocks_to_execute = len(code)

        results = []

        while code_blocks_to_execute > 0:
            try:
                rsp = json.loads(ws.recv())
                msg_type = rsp["msg_type"]
                if msg_type == "error":
                    raise JupyterCellCallError(content=rsp["content"], msg=rsp["content"]["traceback"][0])
            except Exception as _e:
                print(f'ERROR: {_e}')
                break

            if msg_type == "execute_result":
                results.append(rsp["content"])

            if (
                msg_type == "execute_reply"
                and rsp["metadata"].get("status") == "ok"
                and rsp["metadata"].get("dependencies_met", False)
            ):
                code_blocks_to_execute -= 1

        ws.close()

        # Delete the kernel
        url = f"{self.server_host}/api/kernels/{kernel['id']}"
        requests.delete(url, headers=self.headers)
        return self.parse_notebook_results(results)


@pytest.fixture(scope="session")
def jupyter_notebook_server(root_dir):
    server = JupyterNotebookTestServer(notebooks_folder=root_dir)
    try:
        server.start_server()
        yield server
    finally:
        output = server.stop_server()
        logger.info(str(output))

tests/integration_tests/conftest.py
nxexox/pymlup
import logging
import os
import pickle
import signal
import subprocess
import sys
import time

import pytest
import requests

import mlup
from mlup.console_scripts.make_app import make_app
from mlup.constants import ModelDataTransformerType


logger = logging.getLogger('mlup.test')


@pytest.mark.parametrize(
    'custom_params',
    [{}, {'port': 8010}],
    ids=['without_custom_params', 'with_custom_params']
)
@pytest.mark.skip
def test_run_maked_app_from_conf(
    tmp_path_factory,
    pickle_scikit_learn_model_config_yaml,
    scikit_learn_binary_cls_model,
    custom_params
):
    path_to_app = str(tmp_path_factory.getbasetemp() / 'test_run_maked_app_from_conf' /
                      'test_run_maked_app_from_conf.py')
    os.makedirs(os.path.dirname(path_to_app), exist_ok=True)
    make_app(
        path_to_file=path_to_app,
        path_to_src=pickle_scikit_learn_model_config_yaml,
        use_config=True,
        config_fields=custom_params,
        force=True,
    )
    proc = subprocess.Popen(
        sys.executable + ' ' + path_to_app,
        shell=True,
        stderr=subprocess.PIPE,
        universal_newlines=True,
    )
    port = custom_params.get('port', mlup.Config.port)
    i = 0
    response, error = None, None
    while i <= 10 and response is None:
        try:
            response = requests.post(
                f'http://0.0.0.0:{port}/predict',
                json={scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]}
            )
        except requests.ConnectionError as e:
            logger.error(f'Attempt number {i}. Request error {e}.')
            time.sleep(i)
        except Exception as e:
            logger.error(f'Attempt number {i}. Exception {e}.')
            error = e
        else:
            if response.status_code != 200:
                logger.error(response.status_code)
                logger.error(response.text)
        i += 1

    # SIGTERM
    os.kill(proc.pid, signal.SIGKILL)
    if error:
        raise error

    output = proc.stderr.read()
    logger.info(output)
    assert response.status_code == 200
    assert response.json() == {'predict_result': [scikit_learn_binary_cls_model.test_model_response_raw]}


@pytest.mark.parametrize(
    'custom_params',
    [{}, {'port': 8011}],
    ids=['without_custom_params', 'with_custom_params']
)
@pytest.mark.skip
def test_run_maked_app_from_up_bin(tmp_path_factory, scikit_learn_binary_cls_model, custom_params):
    path_to_app = str(tmp_path_factory.getbasetemp() / 'test_run_maked_app_from_up_bin' /
                      'test_run_maked_app_from_up_bin.py')
    path_to_pickle = os.path.join(os.path.dirname(path_to_app), 'test_run_maked_app_from_up_bin.pckl')
    os.makedirs(os.path.dirname(path_to_app), exist_ok=True)

    up = mlup.UP(
        ml_model=scikit_learn_binary_cls_model.model,
        conf=mlup.Config(port=8010, data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
    )
    up.ml.load()
    with open(path_to_pickle, 'wb') as f:
        pickle.dump(up, f)

    make_app(
        path_to_file=path_to_app,
        path_to_src=path_to_pickle,
        use_binary=True,
        config_fields=custom_params,
        force=True,
    )
    proc = subprocess.Popen(
        sys.executable + ' ' + path_to_app,
        shell=True,
        stderr=subprocess.PIPE,
        universal_newlines=True,
    )
    port = custom_params.get('port', up.conf.port)
    i = 0
    response, error = None, None
    while i <= 10 and response is None:
        try:
            response = requests.post(
                f'http://0.0.0.0:{port}/predict',
                json={scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]}
            )
        except requests.ConnectionError as e:
            logger.error(f'Attempt number {i}. Request error {e}.')
            time.sleep(i)
        except Exception as e:
            error = e
        else:
            if response.status_code != 200:
                logger.error(response.status_code)
                logger.error(response.text)
        i += 1

    # SIGTERM
    os.kill(proc.pid, signal.SIGKILL)
    if error:
        raise error

    output = proc.stderr.read()
    logger.info(output)
    assert response.status_code == 200
    assert response.json() == {'predict_result': [scikit_learn_binary_cls_model.test_model_response_raw]}


@pytest.mark.parametrize(
    'custom_params',
    [
        {},
        {
            'port': 8012,
            'data_transformer_for_predict': f"'{ModelDataTransformerType.NUMPY_ARR.value}'"
        }
    ],
    ids=[
        'without_custom_params',
        'with_custom_params'
    ]
)
@pytest.mark.skip
def test_run_maked_app_from_model_bin(tmp_path_factory, scikit_learn_binary_cls_model, custom_params):
    path_to_app = str(tmp_path_factory.getbasetemp() / 'test_run_maked_app_from_model_bin' /
                      'test_run_maked_app_from_model_bin.py')
    os.makedirs(os.path.dirname(path_to_app), exist_ok=True)

    make_app(
        path_to_file=path_to_app,
        path_to_src=scikit_learn_binary_cls_model.path,
        use_model=True,
        config_fields=custom_params,
        force=True,
    )
    proc = subprocess.Popen(
        sys.executable + ' ' + path_to_app,
        shell=True,
        stderr=subprocess.PIPE,
        universal_newlines=True,
    )
    port = custom_params.get('port', mlup.Config.port)
    test_data_raw = scikit_learn_binary_cls_model.test_data_raw
    if custom_params:
        test_data_raw = [1, 2, 3, 4, 5, 6, 7, 8]

    i = 0
    response, error = None, None
    while i <= 10 and response is None:
        try:
            response = requests.post(
                f'http://0.0.0.0:{port}/predict',
                json={scikit_learn_binary_cls_model.x_arg_name: [test_data_raw]}
            )
        except requests.ConnectionError as e:
            logger.error(f'Attempt number {i}. Request error {e}.')
            time.sleep(i)
        except Exception as e:
            error = e
        else:
            if response.status_code != 200:
                logger.error(response.status_code)
                logger.error(response.text)
        i += 1

    # SIGTERM
    os.kill(proc.pid, signal.SIGKILL)
    if error:
        raise error

    output = proc.stderr.read()
    logger.info(output)
    assert response.status_code == 200
    assert response.json() == {"predict_result": [scikit_learn_binary_cls_model.test_model_response_raw]}

tests/integration_tests/console_commands/test_make_app.py
nxexox/pymlup
import asyncio
from asyncio import InvalidStateError
import pickle

import pytest

from mlup.constants import ModelDataTransformerType, BinarizationType, StorageType, WebAppArchitecture
from mlup.up import UP, Config
from mlup.utils.loop import create_async_task

try:
    import lightgbm as lgb
except ImportError:
    lgb = None


@pytest.mark.skipif(lgb is None, reason='lightgbm library not installed.')
class TestLightGBMModel:
    @pytest.mark.parametrize(
        'binarizer_type, model_fixture_name',
        [
            (BinarizationType.PICKLE, 'lightgbm_binary_cls_model'),
            (BinarizationType.LIGHTGBM, 'lightgbm_binary_cls_model_txt'),
        ]
    )
    @pytest.mark.asyncio
    async def test_load_from_source(self, binarizer_type, model_fixture_name, request):
        model_and_path = request.getfixturevalue(model_fixture_name)
        up = UP(
            conf=Config(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': model_and_path.path, 'files_mask': model_and_path.file_mask},
                data_transformer_for_predict=ModelDataTransformerType.PANDAS_DF,
                data_transformer_for_predicted=ModelDataTransformerType.NUMPY_ARR,
                binarization_type=binarizer_type,
            ),
        )
        up.ml.load()
        assert up.predict(**{model_and_path.x_arg_name: [model_and_path.test_data_raw]}) == \
               [model_and_path.test_model_response_raw]

    @pytest.mark.asyncio
    async def test_predict(self, lightgbm_binary_cls_model):
        up = UP(
            ml_model=lightgbm_binary_cls_model.model,
            conf=Config(
                data_transformer_for_predict=ModelDataTransformerType.PANDAS_DF,
                data_transformer_for_predicted=ModelDataTransformerType.NUMPY_ARR,
            )
        )
        up.ml.load()

        pred = up.predict(**{lightgbm_binary_cls_model.x_arg_name: [lightgbm_binary_cls_model.test_data_raw]})
        assert pred == [lightgbm_binary_cls_model.test_model_response_raw]

    @pytest.mark.parametrize(
        'use_thread_loop', [True, False],
        ids=['use_thread_loop=True', 'use_thread_loop=False']
    )
    @pytest.mark.asyncio
    async def test_predict_thread_pool_worker(self, lightgbm_binary_cls_model, use_thread_loop):
        up = UP(
            ml_model=lightgbm_binary_cls_model.model,
            conf=Config(
                use_thread_loop=use_thread_loop,
                data_transformer_for_predict=ModelDataTransformerType.PANDAS_DF,
                data_transformer_for_predicted=ModelDataTransformerType.NUMPY_ARR,
            ),
        )
        up.ml.load()

        pred = up.predict(**{lightgbm_binary_cls_model.x_arg_name: [lightgbm_binary_cls_model.test_data_raw]})
        assert pred == [lightgbm_binary_cls_model.test_model_response_raw]

    @pytest.mark.asyncio
    async def test_web_app_run_and_predict(self, lightgbm_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=lightgbm_binary_cls_model.model,
            conf=Config(
                data_transformer_for_predict=ModelDataTransformerType.PANDAS_DF,
                data_transformer_for_predicted=ModelDataTransformerType.NUMPY_ARR,
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            response = await api_test_client.post(
                "/predict",
                json={lightgbm_binary_cls_model.x_arg_name: [lightgbm_binary_cls_model.test_data_raw]},
            )
            assert response.status_code == 200
            assert response.json() == {"predict_result": [lightgbm_binary_cls_model.test_model_response_raw]}

    @pytest.mark.asyncio
    async def test_web_app_with_worker_architecture(self, lightgbm_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=lightgbm_binary_cls_model.model,
            conf=Config(
                data_transformer_for_predict=ModelDataTransformerType.PANDAS_DF,
                data_transformer_for_predicted=ModelDataTransformerType.NUMPY_ARR,
                mode=WebAppArchitecture.worker_and_queue,
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            response = await api_test_client.post(
                "/predict",
                json={lightgbm_binary_cls_model.x_arg_name: [lightgbm_binary_cls_model.test_data_raw]},
            )
            assert response.status_code == 200
            assert response.json() == {"predict_result": [lightgbm_binary_cls_model.test_model_response_raw]}

    @pytest.mark.asyncio
    async def test_web_app_with_batching_architecture(self, lightgbm_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=lightgbm_binary_cls_model.model,
            conf=Config(
                data_transformer_for_predict=ModelDataTransformerType.PANDAS_DF,
                data_transformer_for_predicted=ModelDataTransformerType.NUMPY_ARR,
                mode=WebAppArchitecture.batching,
                ttl_predicted_data=1,
                batch_worker_timeout=0.1,
                ttl_client_wait=2,
                is_long_predict=True,
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            try:
                await asyncio.sleep(0.05)
                pred_resp_1 = await api_test_client.post(
                    "/predict",
                    json={lightgbm_binary_cls_model.x_arg_name: [lightgbm_binary_cls_model.test_data_raw]},
                )
                await asyncio.sleep(1)
                pred_resp_2 = await api_test_client.post(
                    "/predict",
                    json={lightgbm_binary_cls_model.x_arg_name: [lightgbm_binary_cls_model.test_data_raw]},
                )
                pred_id_2 = pred_resp_2.json()["predict_result"]["predict_id"]
                pred_result_resp_2 = await api_test_client.get("/get-predict/" + pred_id_2)
                assert pred_result_resp_2.json() == [lightgbm_binary_cls_model.test_model_response_raw]

                # Not found results. Long client wait response
                pred_id_1 = pred_resp_1.json()["predict_result"]["predict_id"]
                task = create_async_task(api_test_client.get("/get-predict/" + pred_id_1))
                await asyncio.sleep(0.2)
                if task.cancelled():
                    pytest.fail('Tasks could not have status cancelled.')
                task.cancel()
                task.result()
            except InvalidStateError as e:
                assert str(e) == 'Result is not set.'
            finally:
                await asyncio.sleep(1)

    @pytest.mark.asyncio
    async def test_pickle(self, lightgbm_binary_cls_model):
        up = UP(
            ml_model=lightgbm_binary_cls_model.model,
            conf=Config(
                data_transformer_for_predict=ModelDataTransformerType.PANDAS_DF,
                data_transformer_for_predicted=ModelDataTransformerType.NUMPY_ARR,
            )
        )
        up.ml.load()
        pred_before_pickle = up.predict(
            **{lightgbm_binary_cls_model.x_arg_name: [lightgbm_binary_cls_model.test_data_raw]}
        )

        up_binary_data = pickle.dumps(up)
        up_after_pickle = pickle.loads(up_binary_data)
        pred_after_pickle = up_after_pickle.predict(
            **{lightgbm_binary_cls_model.x_arg_name: [lightgbm_binary_cls_model.test_data_raw]}
        )
        assert pred_after_pickle == pred_before_pickle == [lightgbm_binary_cls_model.test_model_response_raw]

tests/integration_tests/frameworks/test_lightgbm_model.py
nxexox/pymlup
import asyncio
from asyncio import InvalidStateError
import pickle

import pytest

from mlup.constants import ModelDataTransformerType, BinarizationType, StorageType, WebAppArchitecture
from mlup.up import UP, Config
from mlup.utils.loop import create_async_task

try:
    import torch
except ImportError:
    torch = None


@pytest.mark.skipif(torch is None, reason='pytorch library not installed.')
class TestPyTorchModel:
    @pytest.mark.parametrize(
        'binarizer_type, model_fixture_name',
        [
            (BinarizationType.PICKLE, 'pytorch_binary_cls_model'),
            (BinarizationType.TORCH, 'pytorch_binary_cls_model_pth'),
            (BinarizationType.TORCH_JIT, 'pytorch_binary_cls_model_jit'),
        ]
    )
    @pytest.mark.asyncio
    async def test_load_from_source(self, binarizer_type, model_fixture_name, request):
        model_and_path = request.getfixturevalue(model_fixture_name)
        up = UP(
            conf=Config(
                predict_method_name='__call__',
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': model_and_path.path, 'files_mask': model_and_path.file_mask},
                data_transformer_for_predict=ModelDataTransformerType.TORCH_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TORCH_TENSOR,
                binarization_type=binarizer_type,
            ),
        )
        up.ml.load()
        assert up.predict(**{model_and_path.x_arg_name: [model_and_path.test_data_raw]}) == \
               [[model_and_path.test_model_response_raw]]

    @pytest.mark.asyncio
    async def test_predict(self, pytorch_binary_cls_model):
        up = UP(
            ml_model=pytorch_binary_cls_model.model,
            conf=Config(
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TORCH_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TORCH_TENSOR,
            )
        )
        up.ml.load()

        pred = up.predict(**{pytorch_binary_cls_model.x_arg_name: [pytorch_binary_cls_model.test_data_raw]})
        assert pred == [[pytorch_binary_cls_model.test_model_response_raw]]

    @pytest.mark.parametrize(
        'use_thread_loop', [True, False],
        ids=['use_thread_loop=True', 'use_thread_loop=False']
    )
    @pytest.mark.asyncio
    async def test_predict_thread_pool_worker(self, pytorch_binary_cls_model, use_thread_loop):
        up = UP(
            ml_model=pytorch_binary_cls_model.model,
            conf=Config(
                use_thread_loop=use_thread_loop,
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TORCH_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TORCH_TENSOR,
            ),
        )
        up.ml.load()

        pred = up.predict(**{pytorch_binary_cls_model.x_arg_name: [pytorch_binary_cls_model.test_data_raw]})
        assert pred == [[pytorch_binary_cls_model.test_model_response_raw]]

    @pytest.mark.asyncio
    async def test_web_app_run_and_predict(self, pytorch_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=pytorch_binary_cls_model.model,
            conf=Config(
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TORCH_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TORCH_TENSOR,
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            response = await api_test_client.post(
                "/predict",
                json={pytorch_binary_cls_model.x_arg_name: [pytorch_binary_cls_model.test_data_raw]},
            )
            assert response.status_code == 200
            assert response.json() == {"predict_result": [[pytorch_binary_cls_model.test_model_response_raw]]}

    @pytest.mark.asyncio
    async def test_web_app_with_worker_architecture(self, pytorch_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=pytorch_binary_cls_model.model,
            conf=Config(
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TORCH_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TORCH_TENSOR,
                # Web App
                mode=WebAppArchitecture.worker_and_queue
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            response = await api_test_client.post(
                "/predict",
                json={pytorch_binary_cls_model.x_arg_name: [pytorch_binary_cls_model.test_data_raw]},
            )
            assert response.status_code == 200
            assert response.json() == {"predict_result": [[pytorch_binary_cls_model.test_model_response_raw]]}

    @pytest.mark.asyncio
    async def test_web_app_with_batching_architecture(self, pytorch_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=pytorch_binary_cls_model.model,
            conf=Config(
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TORCH_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TORCH_TENSOR,
                # Web App
                mode=WebAppArchitecture.batching,
                ttl_predicted_data=1,
                batch_worker_timeout=0.1,
                ttl_client_wait=2,
                is_long_predict=True,
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            try:
                await asyncio.sleep(0.05)
                pred_resp_1 = await api_test_client.post(
                    "/predict",
                    json={pytorch_binary_cls_model.x_arg_name: [pytorch_binary_cls_model.test_data_raw]},
                )
                await asyncio.sleep(1)
                pred_resp_2 = await api_test_client.post(
                    "/predict",
                    json={pytorch_binary_cls_model.x_arg_name: [pytorch_binary_cls_model.test_data_raw]},
                )
                pred_id_2 = pred_resp_2.json()["predict_result"]["predict_id"]
                pred_result_resp_2 = await api_test_client.get("/get-predict/" + pred_id_2)
                assert pred_result_resp_2.json() == [[pytorch_binary_cls_model.test_model_response_raw]]

                # Not found results. Long client wait response
                pred_id_1 = pred_resp_1.json()["predict_result"]["predict_id"]
                task = create_async_task(api_test_client.get("/get-predict/" + pred_id_1))
                await asyncio.sleep(0.2)
                if task.cancelled():
                    pytest.fail('Tasks could not have status cancelled.')
                task.cancel()
                task.result()
            except InvalidStateError as e:
                assert str(e) == 'Result is not set.'
            finally:
                await asyncio.sleep(1)

    @pytest.mark.asyncio
    async def test_pickle(self, pytorch_binary_cls_model):
        up = UP(
            ml_model=pytorch_binary_cls_model.model,
            conf=Config(
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TORCH_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TORCH_TENSOR,
            )
        )
        up.ml.load()
        pred_before_pickle = up.predict(
            **{pytorch_binary_cls_model.x_arg_name: [pytorch_binary_cls_model.test_data_raw]}
        )

        up_binary_data = pickle.dumps(up)
        up_after_pickle = pickle.loads(up_binary_data)
        pred_after_pickle = up_after_pickle.predict(
            **{pytorch_binary_cls_model.x_arg_name: [pytorch_binary_cls_model.test_data_raw]}
        )
        assert pred_after_pickle == pred_before_pickle == [[pytorch_binary_cls_model.test_model_response_raw]]

tests/integration_tests/frameworks/test_pytorch_model.py
nxexox/pymlup
import asyncio
from asyncio import InvalidStateError
import pickle

import pytest

from mlup.up import UP, Config
from mlup.constants import WebAppArchitecture
from mlup.utils.loop import create_async_task

try:
    import sklearn
except ImportError:
    sklearn = None


@pytest.mark.skipif(sklearn is None, reason='scikit-learn library not installed.')
class TestScikitLearnModel:
    @pytest.mark.asyncio
    async def test_load_from_source(self, scikit_learn_binary_cls_model):
        up = UP(ml_model=scikit_learn_binary_cls_model.model)
        up.ml.load()

        up.predict(**{scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]})

    @pytest.mark.asyncio
    async def test_predict(self, scikit_learn_binary_cls_model):
        up = UP(ml_model=scikit_learn_binary_cls_model.model)
        up.ml.load()

        pred = up.predict(**{scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]})
        assert pred == [scikit_learn_binary_cls_model.test_model_response_raw]

    @pytest.mark.parametrize(
        'use_thread_loop', [True, False],
        ids=['use_thread_loop=True', 'use_thread_loop=False']
    )
    @pytest.mark.asyncio
    async def test_predict_thread_pool_worker(self, scikit_learn_binary_cls_model, use_thread_loop):
        up = UP(
            ml_model=scikit_learn_binary_cls_model.model,
            conf=Config(use_thread_loop=use_thread_loop),
        )
        up.ml.load()

        pred = up.predict(**{scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]})
        assert pred == [scikit_learn_binary_cls_model.test_model_response_raw]

    @pytest.mark.asyncio
    async def test_web_app_run_and_predict(self, scikit_learn_binary_cls_model, web_app_test_client):
        up = UP(ml_model=scikit_learn_binary_cls_model.model)
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            response = await api_test_client.post(
                "/predict",
                json={scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]},
            )
            assert response.status_code == 200
            assert response.json() == {"predict_result": [scikit_learn_binary_cls_model.test_model_response_raw]}

    @pytest.mark.asyncio
    async def test_web_app_with_worker_architecture(self, scikit_learn_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=scikit_learn_binary_cls_model.model,
            conf=Config(
                mode=WebAppArchitecture.worker_and_queue
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            response = await api_test_client.post(
                "/predict",
                json={scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]},
            )
            assert response.status_code == 200
            assert response.json() == {"predict_result": [scikit_learn_binary_cls_model.test_model_response_raw]}

    @pytest.mark.asyncio
    async def test_web_app_with_batching_architecture(self, scikit_learn_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=scikit_learn_binary_cls_model.model,
            conf=Config(
                mode=WebAppArchitecture.batching,
                ttl_predicted_data=1,
                batch_worker_timeout=0.1,
                ttl_client_wait=2,
                is_long_predict=True,
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            try:
                await asyncio.sleep(0.05)
                pred_resp_1 = await api_test_client.post(
                    "/predict",
                    json={scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]},
                )
                await asyncio.sleep(1)
                pred_resp_2 = await api_test_client.post(
                    "/predict",
                    json={scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]},
                )
                pred_id_2 = pred_resp_2.json()["predict_result"]["predict_id"]
                pred_result_resp_2 = await api_test_client.get("/get-predict/" + pred_id_2)
                assert pred_result_resp_2.json() == [scikit_learn_binary_cls_model.test_model_response_raw]

                # Not found results. Long client wait response
                pred_id_1 = pred_resp_1.json()["predict_result"]["predict_id"]
                task = create_async_task(api_test_client.get("/get-predict/" + pred_id_1))
                await asyncio.sleep(0.2)
                if task.cancelled():
                    pytest.fail('Tasks could not have status cancelled.')
                task.cancel()
                task.result()
            except InvalidStateError as e:
                assert str(e) == 'Result is not set.'
            finally:
                await asyncio.sleep(1)

    @pytest.mark.asyncio
    async def test_pickle(self, scikit_learn_binary_cls_model):
        up = UP(ml_model=scikit_learn_binary_cls_model.model)
        up.ml.load()
        pred_before_pickle = up.predict(
            **{scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]}
        )

        up_binary_data = pickle.dumps(up)
        up_after_pickle = pickle.loads(up_binary_data)
        pred_after_pickle = up_after_pickle.predict(
            **{scikit_learn_binary_cls_model.x_arg_name: [scikit_learn_binary_cls_model.test_data_raw]}
        )
        assert pred_after_pickle == pred_before_pickle

tests/integration_tests/frameworks/test_scikit_learn_model.py
nxexox/pymlup
import asyncio
import sys
from asyncio import InvalidStateError
import pickle

import pytest

from mlup.constants import ModelDataTransformerType, StorageType, BinarizationType, WebAppArchitecture
from mlup.up import UP, Config
from mlup.utils.loop import create_async_task

try:
    import tensorflow
except ImportError:
    tensorflow = None


@pytest.mark.skipif(tensorflow is None, reason='tensorflow library not installed.')
class TestTensorFlowModel:
    @pytest.mark.parametrize(
        'binarizer_type, model_fixture_name',
        [
            (BinarizationType.PICKLE, 'tensorflow_binary_cls_model'),
            (BinarizationType.TENSORFLOW, 'tensorflow_binary_cls_model_keras'),
            (BinarizationType.TENSORFLOW, 'tensorflow_binary_cls_model_h5'),
            (BinarizationType.TENSORFLOW_ZIP, 'tensorflow_binary_cls_model_zip'),
        ]
    )
    @pytest.mark.asyncio
    async def test_load_from_source(self, binarizer_type, model_fixture_name, request):
        if sys.version_info.minor == 7 and model_fixture_name == 'tensorflow_binary_cls_model_zip':
            pytest.skip('For Python3.7 keras have version less 2.13 and dont have this model format.')
        model_and_path = request.getfixturevalue(model_fixture_name)
        up = UP(
            conf=Config(
                predict_method_name=model_and_path.predict_method_name,
                storage_type=StorageType.disk,
                storage_kwargs={
                    'path_to_files': str(model_and_path.path),
                    'files_mask': model_and_path.file_mask,
                },
                data_transformer_for_predict=ModelDataTransformerType.TENSORFLOW_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TENSORFLOW_TENSOR,
                binarization_type=binarizer_type,
            ),
        )
        up.ml.load()

        assert up.predict(**{model_and_path.x_arg_name: [model_and_path.test_data_raw]}) == \
               [[model_and_path.test_model_response_raw]]

    @pytest.mark.asyncio
    async def test_predict(self, tensorflow_binary_cls_model):
        up = UP(
            ml_model=tensorflow_binary_cls_model.model,
            conf=Config(
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TENSORFLOW_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TENSORFLOW_TENSOR,
            )
        )
        up.ml.load()

        pred = up.predict(**{tensorflow_binary_cls_model.x_arg_name: [tensorflow_binary_cls_model.test_data_raw]})
        assert pred == [[tensorflow_binary_cls_model.test_model_response_raw]]

    @pytest.mark.parametrize(
        'use_thread_loop', [True, False],
        ids=['use_thread_loop=True', 'use_thread_loop=False']
    )
    @pytest.mark.asyncio
    async def test_predict_thread_pool_worker(self, tensorflow_binary_cls_model, use_thread_loop):
        up = UP(
            ml_model=tensorflow_binary_cls_model.model,
            conf=Config(
                use_thread_loop=use_thread_loop,
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TENSORFLOW_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TENSORFLOW_TENSOR,
            ),
        )
        up.ml.load()

        pred = up.predict(**{tensorflow_binary_cls_model.x_arg_name: [tensorflow_binary_cls_model.test_data_raw]})
        assert pred == [[tensorflow_binary_cls_model.test_model_response_raw]]

    @pytest.mark.asyncio
    async def test_web_app_run_and_predict(self, tensorflow_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=tensorflow_binary_cls_model.model,
            conf=Config(
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TENSORFLOW_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TENSORFLOW_TENSOR,
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            response = await api_test_client.post(
                "/predict",
                json={tensorflow_binary_cls_model.x_arg_name: [tensorflow_binary_cls_model.test_data_raw]},
            )
            assert response.status_code == 200
            assert response.json() == {"predict_result": [[tensorflow_binary_cls_model.test_model_response_raw]]}

    @pytest.mark.asyncio
    async def test_web_app_with_worker_architecture(self, tensorflow_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=tensorflow_binary_cls_model.model,
            conf=Config(
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TENSORFLOW_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TENSORFLOW_TENSOR,
                # Web App
                mode=WebAppArchitecture.worker_and_queue
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            response = await api_test_client.post(
                "/predict",
                json={tensorflow_binary_cls_model.x_arg_name: [tensorflow_binary_cls_model.test_data_raw]},
            )
            assert response.status_code == 200
            assert response.json() == {"predict_result": [[tensorflow_binary_cls_model.test_model_response_raw]]}

    @pytest.mark.asyncio
    async def test_web_app_with_batching_architecture(self, tensorflow_binary_cls_model, web_app_test_client):
        up = UP(
            ml_model=tensorflow_binary_cls_model.model,
            conf=Config(
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TENSORFLOW_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TENSORFLOW_TENSOR,
                # Web App
                mode=WebAppArchitecture.batching,
                ttl_predicted_data=1,
                batch_worker_timeout=0.1,
                ttl_client_wait=2,
                is_long_predict=True,
            )
        )
        up.ml.load()
        up.web.load()

        with web_app_test_client(up.web) as api_test_client:
            try:
                await asyncio.sleep(0.05)
                pred_resp_1 = await api_test_client.post(
                    "/predict",
                    json={tensorflow_binary_cls_model.x_arg_name: [tensorflow_binary_cls_model.test_data_raw]},
                )
                await asyncio.sleep(1)
                pred_resp_2 = await api_test_client.post(
                    "/predict",
                    json={tensorflow_binary_cls_model.x_arg_name: [tensorflow_binary_cls_model.test_data_raw]},
                )
                pred_id_2 = pred_resp_2.json()["predict_result"]["predict_id"]
                pred_result_resp_2 = await api_test_client.get("/get-predict/" + pred_id_2)
                assert pred_result_resp_2.json() == [[tensorflow_binary_cls_model.test_model_response_raw]]

                # Not found results. Long client wait response
                pred_id_1 = pred_resp_1.json()["predict_result"]["predict_id"]
                task = create_async_task(api_test_client.get("/get-predict/" + pred_id_1))
                await asyncio.sleep(0.2)
                if task.cancelled():
                    pytest.fail('Tasks could not have status cancelled.')
                task.cancel()
                task.result()
            except InvalidStateError as e:
                assert str(e) == 'Result is not set.'
            finally:
                await asyncio.sleep(1)

    @pytest.mark.asyncio
    async def test_pickle(self, tensorflow_binary_cls_model):
        up = UP(
            ml_model=tensorflow_binary_cls_model.model,
            conf=Config(
                predict_method_name='__call__',
                data_transformer_for_predict=ModelDataTransformerType.TENSORFLOW_TENSOR,
                data_transformer_for_predicted=ModelDataTransformerType.TENSORFLOW_TENSOR,
            )
        )
        up.ml.load()
        pred_before_pickle = up.predict(
            **{tensorflow_binary_cls_model.x_arg_name: [tensorflow_binary_cls_model.test_data_raw]}
        )

        up_binary_data = pickle.dumps(up)
        up_after_pickle = pickle.loads(up_binary_data)
        pred_after_pickle = up_after_pickle.predict(
            **{tensorflow_binary_cls_model.x_arg_name: [tensorflow_binary_cls_model.test_data_raw]}
        )
        assert pred_after_pickle == pred_before_pickle == [[tensorflow_binary_cls_model.test_model_response_raw]]

tests/integration_tests/frameworks/test_tensorflow_model.py
nxexox/pymlup
import pytest


@pytest.mark.parametrize(
    'test_case',
    [
        'run_easy.ipynb',
        'run_worker_and_queue.ipynb',
        'run_batching.ipynb'
    ],
)
def test_run_web_app(jupyter_notebook_server, tests_jupyter_notebooks_datadir, test_case):
    res = jupyter_notebook_server.run_notebook(tests_jupyter_notebooks_datadir + '/' + test_case)
    # First item is up.predict result, second item is requests.post result
    assert res == ['[1, 2, 3]', "{'predict_result': [1, 2, 3]}"]


def test_call_predict_from(jupyter_notebook_server, tests_jupyter_notebooks_datadir):
    res = jupyter_notebook_server.run_notebook(tests_jupyter_notebooks_datadir + '/call_predict_from.ipynb')
    # First item is up.predict result, second item is requests.post result
    assert res == ['array([1, 2, 3])']

tests/integration_tests/test_jupyter_notebook.py
nxexox/pymlup
import logging
import os
import pickle

import pytest

from mlup.up import UP, Config
from mlup.constants import ModelDataTransformerType, StorageType, BinarizationType, WebAppArchitecture
from mlup.ml.model import MLupModel, ModelConfig

logger = logging.getLogger('mlup.test')
try:
    import joblib
except (ModuleNotFoundError, AttributeError) as e:
    logger.info(f'joblib library not installed. Skip test. {e}')
    joblib = None


@pytest.mark.asyncio
async def test_mlup_model_pickle_serilization(pickle_print_model, models_datadir):
    mlup_model = MLupModel(
        conf=ModelConfig(
            data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            storage_type=StorageType.disk,
            storage_kwargs={'path_to_files': pickle_print_model},
        )
    )
    mlup_model.load()
    pred = await mlup_model.predict(X=[[1, 2, 3]])
    assert pred == [[1, 2, 3]]

    mlup_model_pickle_data = pickle.dumps(mlup_model)
    mlup_unpickle = pickle.loads(mlup_model_pickle_data)

    pred = await mlup_unpickle.predict(X=[[1, 2, 3]])
    assert pred == [[1, 2, 3]]


@pytest.mark.asyncio
@pytest.mark.skipif(joblib is None, reason='joblib library not installed.')
async def test_mlup_model_joblib_serilization(joblib_print_model):
    mlup_model = MLupModel(
        conf=ModelConfig(
            data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            storage_type=StorageType.disk,
            storage_kwargs={'path_to_files': joblib_print_model, 'files_mask': r'(\w.-_)*.joblib'},
            binarization_type=BinarizationType.JOBLIB,
        )
    )
    mlup_model.load()
    pred = await mlup_model.predict(X=[[1, 2, 3]])
    assert pred == [[1, 2, 3]]

    new_model_path = os.path.join(
        str(joblib_print_model).rsplit('/', 1)[0],
        'test_mlup_model_joblib_serilization'
    )
    new_model_name = os.path.join(new_model_path, 'joblib_model.joblib')
    os.makedirs(new_model_path)

    joblib.dump(mlup_model, new_model_name)
    mlup_unjoblib = joblib.load(new_model_name)

    pred = await mlup_unjoblib.predict(X=[[1, 2, 3]])
    assert pred == [[1, 2, 3]]


@pytest.mark.parametrize(
    'mode',
    [
        WebAppArchitecture.directly_to_predict,
        WebAppArchitecture.worker_and_queue,
        WebAppArchitecture.batching,
    ],
    ids=['web.mode=directly_to_predict', 'web.mode=worker_and_queue', 'web.mode=batching']
)
@pytest.mark.asyncio
async def test_mlup_pickle_serilization(pickle_print_model, mode: WebAppArchitecture):
    up = UP(
        conf=Config(
            data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            storage_type=StorageType.disk,
            storage_kwargs={'path_to_files': pickle_print_model},
            mode=mode,
        )
    )
    up.ml.load()
    up.web.load()
    pred = up.predict(X=[[1, 2, 3]])
    assert pred == [[1, 2, 3]]

    up_pickle_data = pickle.dumps(up)
    up_unpickle = pickle.loads(up_pickle_data)

    pred = up_unpickle.predict(X=[[1, 2, 3]])
    assert pred == [[1, 2, 3]]


@pytest.mark.parametrize(
    'mode',
    [
        WebAppArchitecture.directly_to_predict,
        WebAppArchitecture.worker_and_queue,
        WebAppArchitecture.batching,
    ],
    ids=['web.mode=directly_to_predict', 'web.mode=worker_and_queue', 'web.mode=batching']
)
@pytest.mark.asyncio
@pytest.mark.skipif(joblib is None, reason='joblib library not installed.')
async def test_mlup_joblib_serilization(joblib_print_model, mode: WebAppArchitecture):
    up = UP(
        conf=Config(
            data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            storage_type=StorageType.disk,
            storage_kwargs={'path_to_files': joblib_print_model, 'files_mask': r'(\w.-_)*.joblib'},
            binarization_type=BinarizationType.JOBLIB,
            mode=mode,
        )
    )
    up.ml.load()
    up.web.load()
    pred = up.predict(X=[[1, 2, 3]])
    assert pred == [[1, 2, 3]]

    new_up_path = os.path.join(
        str(joblib_print_model).rsplit('/', 1)[0],
        'test_mlup_joblib_serilization'
    )
    new_up_name = os.path.join(new_up_path, 'joblib_up.joblib')
    if os.path.exists(new_up_path):
        new_up_name = new_up_path.rsplit('/', 1)[0] + \
                      new_up_path.rsplit('/', 1)[1] + \
                      f'{len(os.listdir(new_up_path))}_joblib_up.joblib'
    else:
        os.makedirs(new_up_path)

    joblib.dump(up, new_up_name)
    up_unjoblib = joblib.load(new_up_name)

    pred = up_unjoblib.predict(X=[[1, 2, 3]])
    assert pred == [[1, 2, 3]]

tests/integration_tests/test_serilization.py
nxexox/pymlup
import pytest

from mlup.console_scripts.command import run_command


@pytest.mark.parametrize(
    'bash_string',
    ['', ' ', '   ', '              ']
)
def test_run_command_short_command(bash_string):
    try:
        run_command([bash_string])
        pytest.fail('Not raised error')
    except SystemExit as e:
        assert e.code == 1


@pytest.mark.parametrize(
    'bash_string',
    ['mlup -h', 'mlup --help', 'mlup help', 'mlup  -h', 'mlup    help', 'mlup -h --help', 'mlup -h --help help']
)
def test_run_command_help_command(bash_string):
    try:
        run_command(bash_string.split())
        pytest.fail('Not raised error')
    except SystemExit as e:
        assert e.code == 1


@pytest.mark.parametrize(
    'bash_string',
    ['mlup not_exists_command', 'mlup not-exists-command']
)
def test_run_command_not_exists_module(bash_string):
    try:
        run_command(bash_string.split())
        pytest.fail('Not raised error')
    except SystemExit as e:
        assert e.code == 1


@pytest.mark.parametrize(
    'bash_string',
    ['validate_config', 'validate-config']
)
def test_run_command_exists_module(bash_string):
    try:
        run_command(bash_string.split())
        pytest.fail('Not raised error')
    except SystemExit as e:
        # Need additional args for command
        assert e.code == 2

tests/unit_tests/console_scripts/test_commands.py
nxexox/pymlup
import logging
import os
from pathlib import Path

import pytest

try:
    import joblib
except ImportError:
    joblib = None

from mlup.console_scripts.make_app import (
    make_app, save_app,
    get_app_code_from_up_bin, get_app_code_from_conf, get_app_code_without_preload,
    get_app_code_from_model_bin,
    GenerateCodeError, SaveAppError, InputArgsError,
)


logger = logging.getLogger('mlup.test')


APP_CODE_WITHOUT_PRELOAD_TEMPLATE = """import mlup


# You can load the model yourself and pass it to the "ml_model" argument.
# up = mlup.UP(ml_model=my_model, conf=mlup.Config())
up = mlup.UP(
    conf=mlup.Config(
        # Set your config, for work model and get model.
        # You can use storage_type and storage_kwargs for auto_load model from storage.{custom_conf}
    )
)
up.ml.load()
up.web.load()

# If you want to run the application yourself, or add something else to it, use this variable.
# Example with uvicorn: uvicorn {path}:app --host 0.0.0.0 --port 80
app = up.web.app

if __name__ == '__main__':
    up.run_web_app()
"""


GET_APP_CODE_FROM_CONF_TEMPLATE = """import mlup


up = mlup.UP.load_from_{config_type}('{config_path}', load_model=False){custom_conf}
up.ml.load()
up.web.load()

# If you want to run the application yourself, or add something else to it, use this variable.
# Example with uvicorn: uvicorn {path}:app --host 0.0.0.0 --port 80
app = up.web.app

if __name__ == '__main__':
    up.run_web_app()
"""


GET_APP_CODE_FROM_PICKLE_TEMPLATE = """import pickle


with open('{binary_path}', 'rb') as f:
    up = pickle.load(f)
{custom_conf}
if not up.ml.loaded:
    up.ml.load()
up.web.load()

# If you want to run the application yourself, or add something else to it, use this variable.
# Example with uvicorn: uvicorn {path}:app --host 0.0.0.0 --port 80
app = up.web.app

if __name__ == '__main__':
    up.run_web_app()
"""


GET_APP_CODE_FROM_JOBLIB_TEMPLATE = """import joblib


up = joblib.load('{binary_path}')
{custom_conf}
if not up.ml.loaded:
    up.ml.load()
up.web.load()

# If you want to run the application yourself, or add something else to it, use this variable.
# Example with uvicorn: uvicorn {path}:app --host 0.0.0.0 --port 80
app = up.web.app

if __name__ == '__main__':
    up.run_web_app()
"""


GET_APP_CODE_FROM_MODEL_BIN_TEMPLATE = """import mlup
from mlup import constants


up = mlup.UP(
    conf=mlup.Config(
        # Set your config, for work model and get model.
        storage_type=constants.StorageType.disk,
        storage_kwargs={{
            'path_to_files': '{model_path}',
            'files_mask': '{model_name}',
        }},{custom_conf}
    )
)
up.ml.load()
up.web.load()

# If you want to run the application yourself, or add something else to it, use this variable.
# Example with uvicorn: uvicorn {path}:app --host 0.0.0.0 --port 80
app = up.web.app

if __name__ == '__main__':
    up.run_web_app()
"""


PATCHES_TO_FILE = [
    'app.py',
    'folder/pup.py',
    'folder/folders/folderssasdasd/test.py/op.py',
    'app',
    'folder/pup',
    'folder/folders/folderssasdasd/test.py/op',
    'app.exe',
    'folder/pup.exe',
    'folder/folders/folderssasdasd/test.py/op.exe',
]
CONF_KWARGS_DICT = {
    'port': 8011,
    'batch_worker_timeout': 10.0,
    'predict_method_name': "'__call__'",
    'use_thread_loop': False,
    'columns': [{"name": "col", "type": "list"}],
    'uvicorn_kwargs': {'workers': 4, 'timeout_graceful_shutdown': 10},
}


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
@pytest.mark.parametrize('with_custom_conf', [False, True])
def test_get_app_code_without_preload(path, with_custom_conf):
    if with_custom_conf:
        kwargs_str = ''
        for k, v in CONF_KWARGS_DICT.items():
            kwargs_str += f"\n        {k}={v},"

        generated_code = get_app_code_without_preload(path, **CONF_KWARGS_DICT)
        expected_code = APP_CODE_WITHOUT_PRELOAD_TEMPLATE.format(
            path=Path(path).stem, custom_conf=kwargs_str
        )
    else:
        generated_code = get_app_code_without_preload(path)
        expected_code = APP_CODE_WITHOUT_PRELOAD_TEMPLATE.format(path=Path(path).stem, custom_conf='')
    assert generated_code == expected_code


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
@pytest.mark.parametrize(
    'config_path, config_type',
    [('not_exists_path.json', 'json'), ('not_exists_path.yaml', 'yaml')],
    ids=['json', 'yaml']
)
def test_get_app_code_from_conf_not_exists_conf(path, config_path, config_type):
    assert get_app_code_from_conf(path, config_path, config_type) == GET_APP_CODE_FROM_CONF_TEMPLATE.format(
        path=Path(path).stem,
        config_path=config_path,
        config_type=config_type,
        custom_conf='',
    )


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
def test_get_app_code_from_conf_not_exists_type(path):
    try:
        get_app_code_from_conf(Path(path).stem, 'not_exists_conf_path', 'not_exists_type')
        pytest.fail('Not raised error')
    except GenerateCodeError as e:
        assert str(e) == 'Config type not_exists_type not supported.'


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
@pytest.mark.parametrize(
    'config_path, config_type',
    [('test_json_config', 'json'), ('test_yaml_config', 'yaml')],
    ids=['json', 'yaml']
)
@pytest.mark.parametrize('with_custom_conf', [False, True])
def test_get_app_code_from_conf_success(path, config_path, config_type, with_custom_conf):
    if with_custom_conf:
        kwargs_str = ''
        for k, v in CONF_KWARGS_DICT.items():
            kwargs_str += f"\nup.conf.{k} = {v}"

        generated_code = get_app_code_from_conf(path, config_path, config_type, **CONF_KWARGS_DICT)
        expected_code = GET_APP_CODE_FROM_CONF_TEMPLATE.format(
            path=Path(path).stem,
            config_type=config_type,
            config_path=config_path,
            custom_conf=kwargs_str,
        )
    else:
        generated_code = get_app_code_from_conf(path, config_path, config_type)
        expected_code = GET_APP_CODE_FROM_CONF_TEMPLATE.format(
            path=Path(path).stem,
            config_type=config_type,
            config_path=config_path,
            custom_conf='',
        )

    assert generated_code == expected_code


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
def test_get_app_code_from_up_bin_not_exists_file(path):
    assert get_app_code_from_up_bin(path, 'not_exist_path', 'pickle') == GET_APP_CODE_FROM_PICKLE_TEMPLATE.format(
        path=Path(path).stem,
        binary_path='not_exist_path',
        custom_conf='',
    )


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
def test_get_app_code_from_up_bin_not_exists_type(path):
    try:
        get_app_code_from_up_bin(Path(path).stem, 'not_exists_path', 'not_exists_type')
        pytest.fail('Not raised error')
    except GenerateCodeError as e:
        assert str(e) == 'Binary type not_exists_type not supported.'


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
@pytest.mark.parametrize(
    'template, binary_path, binary_type',
    [
        (GET_APP_CODE_FROM_PICKLE_TEMPLATE, 'binary.pckl', 'pickle'),
        (GET_APP_CODE_FROM_JOBLIB_TEMPLATE, 'binary.joblib', 'joblib')
    ],
    ids=['pickle', 'joblib']
)
@pytest.mark.parametrize('with_custom_conf', [False, True])
def test_get_app_code_from_up_bin_success(path, template, binary_path, binary_type, with_custom_conf):
    if with_custom_conf:
        kwargs_str = ''
        for k, v in CONF_KWARGS_DICT.items():
            kwargs_str += f"\nup.conf.{k} = {v}"

        generated_code = get_app_code_from_up_bin(path, binary_path, binary_type, **CONF_KWARGS_DICT)
        expected_code = template.format(
            path=Path(path).stem,
            binary_path=binary_path,
            custom_conf=kwargs_str,
        )
    else:
        generated_code = get_app_code_from_up_bin(path, binary_path, binary_type)
        expected_code = template.format(path=Path(path).stem, binary_path=binary_path, custom_conf='')

    assert generated_code == expected_code


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
def test_get_app_code_from_model_bin_not_exists_file(path):
    assert get_app_code_from_model_bin(path, 'not_exist_model.onnx') == GET_APP_CODE_FROM_MODEL_BIN_TEMPLATE.format(
        path=Path(path).stem,
        model_path='not_exist_model.onnx',
        model_name='not_exist_model.onnx',
        custom_conf='',
    )


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
@pytest.mark.parametrize('with_custom_conf', [False, True])
def test_get_app_code_from_model_bin_success(path, with_custom_conf, pickle_print_model):
    if with_custom_conf:
        kwargs_str = ''
        for k, v in CONF_KWARGS_DICT.items():
            kwargs_str += f"\n        {k}={v},"

        generated_code = get_app_code_from_model_bin(path, pickle_print_model, **CONF_KWARGS_DICT)
        expected_code = GET_APP_CODE_FROM_MODEL_BIN_TEMPLATE.format(
            path=Path(path).stem,
            model_path=pickle_print_model,
            model_name=Path(pickle_print_model).name,
            custom_conf=kwargs_str,
        )
    else:
        generated_code = get_app_code_from_model_bin(path, pickle_print_model)
        expected_code = GET_APP_CODE_FROM_MODEL_BIN_TEMPLATE.format(
            path=Path(path).stem,
            model_path=pickle_print_model,
            model_name=Path(pickle_print_model).name,
            custom_conf=''
        )

    assert generated_code == expected_code


@pytest.mark.parametrize('force', [False, True])
@pytest.mark.parametrize(
    'app_name, app_code',
    [(p, 'test app code') for p in PATCHES_TO_FILE],
    ids=PATCHES_TO_FILE
)
def test_save_app_exists_file(tmp_path_factory, app_name, app_code, force):
    path = tmp_path_factory.getbasetemp() / 'test_save_app_exists_file' / app_name
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, 'w') as f:
        f.write(app_code)

    try:
        save_app(str(path), app_code, force)
        if force is True:
            with open(path, 'r') as f:
                assert f.read() == app_code
        else:
            pytest.fail('Not raised error')
    except SaveAppError as e:
        if force is True:
            pytest.fail(f'Raise error {e}')
        else:
            assert str(e) == f'File "{path}" exists. For replace, use argument force=True'


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
def test_make_app_with_config_and_binary(path):
    try:
        make_app(
            path_to_file=path,
            use_config=True,
            use_binary=True,
        )
        pytest.fail('Not raised error')
    except InputArgsError as e:
        assert str(e) == 'You can specify only one of the arguments: use_config [-c], use_binary [-b], use_model [-m].'


@pytest.mark.parametrize('path', PATCHES_TO_FILE)
def test_make_app_without_config_and_binary(path):
    try:
        make_app(
            path_to_file=path,
            use_config=True,
            path_to_src=None,
        )
        pytest.fail('Not raised error')
    except InputArgsError as e:
        assert str(e) == 'For arguments use_config [-c], use_binary [-b], use_model [-m] need set argument path_to_src [-s].'


def test_make_app_with_not_exists_src_path():
    try:
        make_app(
            path_to_file='nox_exists_path',
            use_config=True,
            path_to_src='not_exists_src_path',
        )
    except FileNotFoundError as e:
        assert str(e) == 'File "not_exists_src_path" not exists.'


def test_make_app_with_exists_path(tmp_path_factory):
    path = tmp_path_factory.getbasetemp() / 'test_make_app_with_exists_path' / \
           'test_run_with_not_exists_path.py'
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, 'w') as f:
        f.write('')

    try:
        make_app(path_to_file=str(path))
    except FileExistsError as e:
        assert str(e) == f'File "{path}" exists.'


def test_make_app_change_extension(tmp_path_factory):
    path = tmp_path_factory.getbasetemp() / 'test_make_app_change_extension' / 'test_make_app_change_extension'
    os.makedirs(os.path.dirname(path), exist_ok=True)
    make_app(path_to_file=str(path))
    assert os.path.exists(str(path) + '.py')
    assert not os.path.exists(path)

    path = tmp_path_factory.getbasetemp() / 'test_make_app_change_extension' / 'test_make_app_change_extension_2'
    os.makedirs(os.path.dirname(path), exist_ok=True)
    make_app(path_to_file=str(path), extension='.exe')
    assert os.path.exists(str(path) + '.exe')
    assert not os.path.exists(path)

    path = tmp_path_factory.getbasetemp() / 'test_make_app_change_extension' / 'test_make_app_change_extension_3.py'
    os.makedirs(os.path.dirname(path), exist_ok=True)
    make_app(path_to_file=str(path))
    assert os.path.exists(str(path))

tests/unit_tests/console_scripts/test_make_app.py
nxexox/pymlup
import json
import os
import pickle
import time
from multiprocessing import Process

import pytest
import requests

from mlup.constants import BinarizationType
from mlup.errors import ModelLoadError, ModelBinarizationError

try:
    import joblib
except ImportError:
    joblib = None

import mlup
from mlup.console_scripts.run import run, run_from_up_bin, run_from_config, run_from_model, InputArgsError


@pytest.mark.parametrize(
    'config_path, config_type',
    [('not_exists_path.json', 'json'), ('not_exists_path.yaml', 'yaml')],
    ids=['json', 'yaml']
)
def test_run_from_config_not_exists_conf(config_path, config_type):
    try:
        run_from_config(config_path, config_type)
        pytest.fail('Not raised error')
    except FileNotFoundError as e:
        assert str(e) == f"[Errno 2] No such file or directory: '{config_path}'"


def test_run_from_config_not_exists_conf_type(tmp_path_factory):
    file_path = tmp_path_factory.getbasetemp() / 'test_run_from_config_not_exists_conf_type' / \
                'test_run_from_config_not_exists_conf_type.json'
    os.makedirs(os.path.dirname(file_path), exist_ok=True)
    with open(file_path, 'w') as f:
        f.write('Not valid config\n\nBut is valid multirows string.')

    try:
        run_from_config(str(file_path), 'not_exists_type')
        pytest.fail('Not raised error')
    except ValueError as e:
        assert str(e) == 'Config type not_exists_type not supported.'


@pytest.mark.parametrize('config_type', ['json', 'yaml'])
def test_run_from_config_not_valid_config(tmp_path_factory, config_type):
    file_path = tmp_path_factory.getbasetemp() / 'test_run_from_config_not_valid_config' / \
                f'test_run_from_config_not_valid_config.{config_type}'
    os.makedirs(os.path.dirname(file_path), exist_ok=True)
    with open(file_path, 'w') as f:
        f.write('Not valid config\n\nBut is valid multirows string.')

    try:
        run_from_config(str(file_path), config_type)
        pytest.fail('Not raised error')
    except json.JSONDecodeError as e:
        assert str(e) == 'Expecting value: line 1 column 1 (char 0)'
    except AttributeError as e:
        assert str(e) == "'str' object has no attribute 'get'"


@pytest.mark.parametrize(
    'config_fixture_name, conf_type',
    [('test_json_config', 'json'), ('test_yaml_config', 'yaml')],
    ids=['json', 'yaml']
)
@pytest.mark.asyncio
async def test_run_from_config_valid_config(request, pickle_print_model, config_fixture_name, conf_type):
    config_path = request.getfixturevalue(config_fixture_name)
    if conf_type == 'json':
        _up = mlup.UP.load_from_json(config_path, load_model=False)
        _up.conf.uvicorn_kwargs['loop'] = 'none'
        _up.conf.storage_kwargs['path_to_files'] = str(pickle_print_model)
        _up.to_json(config_path)
    elif conf_type == 'yaml':
        _up = mlup.UP.load_from_yaml(config_path, load_model=False)
        _up.conf.uvicorn_kwargs['loop'] = 'none'
        _up.conf.storage_kwargs['path_to_files'] = str(pickle_print_model)
        _up.to_yaml(config_path)
    else:
        pytest.fail(f'Not supported config type {conf_type}')

    proc = Process(
        target=run_from_config,
        args=(config_path, conf_type),
        daemon=False
    )
    try:
        proc.start()
        time.sleep(10)

        resp = requests.get('http://0.0.0.0:8009/health')
        assert resp.status_code == 200
        assert resp.json() == {'status': 200}
    finally:
        proc.terminate()
        time.sleep(3)


@pytest.mark.parametrize(
    'binary_path, binary_type',
    [('not_exists_path.pckl', 'pickle'), ('not_exists_path.joblib', 'joblib')],
    ids=['pickle', 'joblib']
)
def test_run_from_up_bin_not_exists_bin(binary_path, binary_type):
    try:
        run_from_up_bin(binary_path, binary_type)
        pytest.fail('Not raised error')
    except FileNotFoundError as e:
        assert str(e) == f"[Errno 2] No such file or directory: '{binary_path}'"


def test_run_from_up_bin_not_exists_binary_type(tmp_path_factory):
    file_path = tmp_path_factory.getbasetemp() / 'test_run_from_up_bin_not_exists_binary_type' / \
                'test_run_from_up_bin_not_exists_binary_type.pckl'
    os.makedirs(os.path.dirname(file_path), exist_ok=True)
    with open(file_path, 'w') as f:
        f.write('Not valid binary\n\nBut is valid multirows string.')

    try:
        run_from_up_bin(str(file_path), 'not_exists_type')
        pytest.fail('Not raised error')
    except ValueError as e:
        assert str(e) == 'Binary type not_exists_type not supported.'


@pytest.mark.parametrize('binary_type', ['pickle', 'joblib'])
def test_run_from_up_bin_not_valid_binary(tmp_path_factory, binary_type):
    file_path = tmp_path_factory.getbasetemp() / 'test_run_from_up_bin_not_valid_binary' / \
                f'test_run_from_up_bin_not_valid_binary.{binary_type}'
    os.makedirs(os.path.dirname(file_path), exist_ok=True)
    with open(file_path, 'w') as f:
        f.write('Not valid binary data\n\nBut is valid multirows string.')

    try:
        run_from_up_bin(str(file_path), binary_type)
        pytest.fail('Not raised error')
    except pickle.UnpicklingError as e:
        assert str(e) == 'could not find MARK'
    except IndexError as e:
        assert str(e) == 'pop from empty list'


@pytest.mark.asyncio
async def test_run_from_up_bin_valid_binary_pickle(tmp_path_factory, print_model):
    full_path = str(tmp_path_factory.getbasetemp() / 'test_run_from_up_bin_valid_binary_pickle' /
                    'test_run_from_up_bin_valid_binary_pickle.pickle')
    os.makedirs(os.path.dirname(full_path), exist_ok=True)
    up = mlup.UP(ml_model=print_model)
    up.ml.load()
    up.conf.uvicorn_kwargs['loop'] = 'none'
    with open(full_path, 'wb') as f:
        pickle.dump(up, f)

    proc = Process(
        target=run_from_up_bin,
        args=(full_path, 'pickle'),
        daemon=False
    )
    try:
        proc.start()
        time.sleep(10)

        resp = requests.get('http://0.0.0.0:8009/health')
        assert resp.status_code == 200
        assert resp.json() == {'status': 200}
    finally:
        proc.terminate()
        time.sleep(3)


@pytest.mark.skipif(joblib is None, reason='joblib library not installed.')
@pytest.mark.asyncio
async def test_run_from_up_bin_valid_binary_joblib(tmp_path_factory, print_model):
    full_path = str(tmp_path_factory.getbasetemp() / 'test_run_from_up_bin_valid_binary_joblib' /
                    'test_run_from_bin_valid_binary_joblib.joblib')
    os.makedirs(os.path.dirname(full_path), exist_ok=True)
    up = mlup.UP(ml_model=print_model)
    up.ml.load()
    up.conf.uvicorn_kwargs['loop'] = 'none'
    joblib.dump(up, full_path)

    proc = Process(
        target=run_from_up_bin,
        args=(full_path, 'joblib'),
        daemon=False
    )
    try:
        proc.start()
        time.sleep(10)

        resp = requests.get('http://0.0.0.0:8009/health')
        assert resp.status_code == 200
        assert resp.json() == {'status': 200}
    finally:
        proc.terminate()
        time.sleep(3)


def test_run_from_model_not_exists_model():
    try:
        run_from_model('not_exists_model.pckl')
        pytest.fail('Not raised error')
    except ModelLoadError as e:
        assert str(e) == "[Errno 2] No such file or directory: 'not_exists_model.pckl'"


def test_run_from_model_not_valid_model(tmp_path_factory):
    file_path = tmp_path_factory.getbasetemp() / 'test_run_from_model_not_valid_model' / \
                'test_run_from_model_not_valid_model.pckl'
    os.makedirs(os.path.dirname(file_path), exist_ok=True)
    with open(file_path, 'w') as f:
        f.write('Not valid model\n\nBut is valid multirows string.')

    try:
        run_from_model(str(file_path), binarization_type=BinarizationType.PICKLE)
        pytest.fail('Not raised error')
    except ModelBinarizationError as e:
        assert str(e) == 'Error with deserialize model: could not find MARK'


@pytest.mark.asyncio
async def test_run_from_model_valid_binary_pickle(pickle_print_model):
    proc = Process(
        target=run_from_model,
        kwargs={'model_path': pickle_print_model, 'port': 8011},
        daemon=False
    )
    try:
        proc.start()
        time.sleep(20)

        resp = requests.get('http://0.0.0.0:8011/health')
        assert resp.status_code == 200
        assert resp.json() == {'status': 200}
    finally:
        proc.terminate()
        time.sleep(3)


def test_run_with_conf_and_bin():
    try:
        run(
            path='',
            use_conf=True,
            use_bin=True
        )
        pytest.fail('Not raised error')
    except InputArgsError as e:
        assert str(e) == 'You can specify only one of the arguments: use_conf [-c], use_bin [-b], use_model [-m].'


def test_run_without_conf_and_bin():
    try:
        run(
            path='',
            use_conf=False,
            use_bin=False,
        )
    except InputArgsError as e:
        assert str(e) == 'You must specify one of the arguments: use_conf [-c], use_bin [-b], use_model [-m].'


def test_run_with_not_exists_path():
    try:
        run(
            path='nox_exists_path',
            use_conf=True,
        )
    except FileNotFoundError as e:
        assert str(e) == 'Path nox_exists_path not exists.'

tests/unit_tests/console_scripts/test_run.py
nxexox/pymlup
import json

import pytest

from mlup.console_scripts.validate_config import validate_config, ValidationConfigError


@pytest.mark.parametrize(
    'config_path, config_type',
    [('not_exists_path.json', 'json'), ('not_exists_path.yaml', 'yaml')],
    ids=['json', 'yaml']
)
def test_validate_config_not_exists_conf(config_path, config_type):
    try:
        validate_config(config_path, config_type)
        pytest.fail('Not raised error')
    except FileNotFoundError as e:
        assert str(e) == f'File "{config_path}" not found.'


def test_validate_config_not_exists_conf_type(tmp_path_factory):
    file_path = tmp_path_factory.getbasetemp() / 'test_validate_config_not_exists_conf_type.json'
    with open(file_path, 'w') as f:
        f.write('Not valid config\n\nBut is valid multirows string.')

    try:
        validate_config(str(file_path), 'not_exists_type')
        pytest.fail('Not raised error')
    except ValidationConfigError as e:
        assert str(e) == 'Config type not_exists_type not supported.'


@pytest.mark.parametrize('config_type', ['json', 'yaml'])
def test_validate_config_not_valid_config(tmp_path_factory, config_type):
    file_path = tmp_path_factory.getbasetemp() / f'test_validate_config_not_valid_config.{config_type}'
    with open(file_path, 'w') as f:
        f.write('Not valid config\n\nBut is valid multirows string.')

    try:
        validate_config(str(file_path), config_type)
        pytest.fail('Not raised error')
    except json.JSONDecodeError as e:
        assert str(e) == 'Expecting value: line 1 column 1 (char 0)'
    except AttributeError as e:
        assert str(e) == "'str' object has no attribute 'get'"


@pytest.mark.parametrize(
    'config_fixture_name, conf_type',
    [('test_json_config', 'json'), ('test_yaml_config', 'yaml')],
    ids=['json', 'yaml']
)
def test_validate_config_valid_config(request, config_fixture_name, conf_type):
    config_path = request.getfixturevalue(config_fixture_name)
    validate_config(config_path, conf_type)

tests/unit_tests/console_scripts/test_validate_config.py
nxexox/pymlup
import logging
from io import BytesIO

import pytest

from mlup.constants import BinarizationType, LoadedFile
from mlup.ml.binarization.memory import MemoryBinarizer
from mlup.ml.binarization.pickle import PickleBinarizer
from mlup.utils.interspection import get_class_by_path

logger = logging.getLogger('mlup.test')
try:
    from mlup.ml.binarization.joblib import JoblibBinarizer
except (ModuleNotFoundError, AttributeError) as e:
    logger.info(f'joblib library not installed. Skip test. {e}')
    JoblibBinarizer = None
try:
    from mlup.ml.binarization.lightgbm import LightGBMBinarizer
except (ModuleNotFoundError, AttributeError) as e:
    logger.info(f'lightgbm library not installed. Skip test. {e}')
    LightGBMBinarizer = None
try:
    from mlup.ml.binarization.onnx import InferenceSessionBinarizer, InferenceSessionFullReturnBinarizer
except (ModuleNotFoundError, AttributeError) as e:
    logger.info(f'onnxruntime library not installed. Skip test. {e}')
    InferenceSessionBinarizer = None
try:
    from mlup.ml.binarization.tensorflow import TensorFlowBinarizer, TensorFlowSavedBinarizer
except (ModuleNotFoundError, AttributeError) as e:
    logger.info(f'tensorflow library not installed. Skip test. {e}')
    TensorFlowBinarizer, TensorFlowSavedBinarizer = None, None
try:
    from mlup.ml.binarization.torch import TorchBinarizer, TorchJITBinarizer
except (ModuleNotFoundError, AttributeError) as e:
    logger.info(f'pytorch library not installed. Skip test. {e}')
    TorchBinarizer, TorchJITBinarizer = None, None


@pytest.fixture(scope="session")
def model_not_readable_from_disk(tmp_path_factory):
    f_path = tmp_path_factory.getbasetemp()
    model_name = 'model_not_readable_from_disk.bin'
    logger.info(f'Create {f_path}/{model_name} fixture.')
    with open(f_path / model_name, 'w') as f:
        f.write(model_name)
    return f_path / model_name


def test_single_file_memory_binarization_deserialize(print_model):
    binarizer = MemoryBinarizer
    model = binarizer.deserialize(LoadedFile(print_model))

    src_x = [1, 2, 3]
    pred_d = model.predict(src_x, True, 4, 5, 6, seven=7, eight=8)
    assert pred_d == src_x


def test_single_file_pickle_binarization_deserialize(pickle_print_model):
    binarizer = PickleBinarizer
    with open(pickle_print_model, 'rb') as f:
        data = f.read()

    model = binarizer.deserialize(LoadedFile(data, pickle_print_model))

    src_x = [1, 2, 3]
    pred_d = model.predict(src_x, True, 4, 5, 6, seven=7, eight=8)
    assert pred_d == src_x


@pytest.mark.parametrize(
    'data',
    [
        pytest.param(
            b'\x80\x04\x95$\x00\x00\x00\x00\x00\x00\x00\x8c\x0etests.conftest\x94\x8c\nPrintModel\x94\x93\x94)\x81\x94.',
            id='bytes'
        ),
        pytest.param(
            BytesIO(b'\x80\x04\x95$\x00\x00\x00\x00\x00\x00\x00\x8c\x0etests.conftest\x94\x8c\nPrintModel\x94\x93\x94)'
                    b'\x81\x94.'),
            id='BufferedReader'
        )
    ]
)
@pytest.mark.skipif(JoblibBinarizer is None, reason='joblib library not installed.')
def test_single_file_joblib_binarization_deserialize(data):
    binarizer = JoblibBinarizer

    model = binarizer.deserialize(LoadedFile(data))

    src_x = [1, 2, 3]
    pred_d = model.predict(src_x, True, 4, 5, 6, seven=7, eight=8)
    assert pred_d == src_x


@pytest.mark.skipif(InferenceSessionBinarizer is None, reason='sklearn, onnxruntime libraries not installed.')
def test_single_file_sklearn_onnx_binarization_deserialize(scikit_learn_binary_cls_model_onnx):
    import numpy as np
    binarizer = InferenceSessionBinarizer
    test_data_raw = np.array([list(scikit_learn_binary_cls_model_onnx.test_data_raw.values())], dtype=np.float32)

    with open(scikit_learn_binary_cls_model_onnx.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(f.read(), scikit_learn_binary_cls_model_onnx.path))
        pred_d = model.predict(test_data_raw)
        assert pred_d[0][0] == scikit_learn_binary_cls_model_onnx.test_model_response_raw

    with open(scikit_learn_binary_cls_model_onnx.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(BytesIO(f.read()), scikit_learn_binary_cls_model_onnx.path))
        pred_d = model.predict(test_data_raw)
        assert pred_d[0][0] == scikit_learn_binary_cls_model_onnx.test_model_response_raw


@pytest.mark.skipif(InferenceSessionBinarizer is None, reason='sklearn, onnxruntime libraries not installed.')
def test_single_file_sklearn_onnx_full_return_binarization_deserialize(scikit_learn_binary_cls_model_onnx):
    import numpy as np
    binarizer = InferenceSessionFullReturnBinarizer
    test_data_raw = np.array([list(scikit_learn_binary_cls_model_onnx.test_data_raw.values())], dtype=np.float32)

    with open(scikit_learn_binary_cls_model_onnx.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(f.read(), scikit_learn_binary_cls_model_onnx.path))
        pred_d = model.predict(test_data_raw)
        assert pred_d[0][0] == scikit_learn_binary_cls_model_onnx.test_model_response_raw
        assert pred_d[1] == [{'No': 1.0, 'Yes': 0.0}]

    with open(scikit_learn_binary_cls_model_onnx.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(BytesIO(f.read()), scikit_learn_binary_cls_model_onnx.path))
        pred_d = model.predict(test_data_raw)
        assert pred_d[0][0] == scikit_learn_binary_cls_model_onnx.test_model_response_raw
        assert pred_d[1] == [{'No': 1.0, 'Yes': 0.0}]


@pytest.mark.skipif(LightGBMBinarizer is None, reason='lightgbm library not installed.')
def test_single_file_lightgbm_binarization_deserialize(lightgbm_binary_cls_model_txt):
    import pandas
    import numpy as np
    binarizer = LightGBMBinarizer
    test_data_raw = pandas.DataFrame(
        [lightgbm_binary_cls_model_txt.test_data_raw],
        columns=list(lightgbm_binary_cls_model_txt.test_data_raw.keys())
    )

    with open(lightgbm_binary_cls_model_txt.path, 'r') as f:
        model = binarizer.deserialize(LoadedFile(f.read(), lightgbm_binary_cls_model_txt.path))
        pred_d = model.predict(test_data_raw)
        assert np.array_equal(
            pred_d,
            np.array([lightgbm_binary_cls_model_txt.test_model_response_raw])
        )

    with open(lightgbm_binary_cls_model_txt.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(BytesIO(f.read()), lightgbm_binary_cls_model_txt.path))
        pred_d = model.predict(test_data_raw)
        assert np.array_equal(
            pred_d,
            np.array([lightgbm_binary_cls_model_txt.test_model_response_raw])
        )


@pytest.mark.parametrize(
    'fixture_name',
    ['tensorflow_binary_cls_model_keras', 'tensorflow_binary_cls_model_h5']
)
@pytest.mark.skipif(TensorFlowBinarizer is None, reason='tensorflow library not installed.')
def test_single_file_tensorflow_binarization_deserialize(fixture_name, request):
    import tensorflow as tf
    binarizer = TensorFlowBinarizer
    base_model = request.getfixturevalue(fixture_name)
    test_data_raw = tf.convert_to_tensor([list(base_model.test_data_raw.values())])

    with open(base_model.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(f.read(), base_model.path))
        pred_d = model(test_data_raw)
        assert pred_d.numpy().tolist() == [[base_model.test_model_response_raw]]

    with open(base_model.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(BytesIO(f.read()), base_model.path))
        pred_d = model(test_data_raw)
        assert pred_d.numpy().tolist() == [[base_model.test_model_response_raw]]


@pytest.mark.skipif(TorchBinarizer is None, reason='pytorch library not installed.')
def test_single_file_torch_binarization_deserialize(pytorch_binary_cls_model_pth):
    import torch
    binarizer = TorchBinarizer
    test_data_raw = torch.tensor(list(pytorch_binary_cls_model_pth.test_data_raw.values()))

    with open(pytorch_binary_cls_model_pth.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(f.read(), pytorch_binary_cls_model_pth.path))
        pred_d = model(test_data_raw)
        assert pred_d == pytorch_binary_cls_model_pth.test_model_response_raw

    with open(pytorch_binary_cls_model_pth.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(BytesIO(f.read()), pytorch_binary_cls_model_pth.path))
        pred_d = model(test_data_raw)
        assert pred_d == pytorch_binary_cls_model_pth.test_model_response_raw


@pytest.mark.skipif(InferenceSessionBinarizer is None, reason='pytorch, onnxruntime libraries not installed.')
def test_single_file_torch_onnx_binarization_deserialize(pytorch_binary_cls_model_onnx):
    import numpy as np
    binarizer = InferenceSessionBinarizer
    test_data_raw = np.array([list(pytorch_binary_cls_model_onnx.test_data_raw.values())], dtype=np.float32)

    with open(pytorch_binary_cls_model_onnx.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(f.read(), pytorch_binary_cls_model_onnx.path))
        pred_d = model.predict(test_data_raw)
        assert pred_d[0].tolist()[0][0] == pytorch_binary_cls_model_onnx.test_model_response_raw

    with open(pytorch_binary_cls_model_onnx.path, 'rb') as f:
        model = binarizer.deserialize(LoadedFile(BytesIO(f.read()), pytorch_binary_cls_model_onnx.path))
        pred_d = model.predict(test_data_raw)
        assert pred_d[0].tolist()[0][0] == pytorch_binary_cls_model_onnx.test_model_response_raw


@pytest.mark.parametrize(
    'binarization_type, binarizer', [
        (BinarizationType.MEMORY, MemoryBinarizer),
        (BinarizationType.MEMORY.value, MemoryBinarizer),
        (BinarizationType.PICKLE, PickleBinarizer),
        (BinarizationType.PICKLE.value, PickleBinarizer),
        (BinarizationType.JOBLIB, JoblibBinarizer),
        (BinarizationType.JOBLIB.value, JoblibBinarizer),
        (BinarizationType.LIGHTGBM, LightGBMBinarizer),
        (BinarizationType.LIGHTGBM.value, LightGBMBinarizer),
        (BinarizationType.TENSORFLOW, TensorFlowBinarizer),
        (BinarizationType.TENSORFLOW.value, TensorFlowBinarizer),
        (BinarizationType.TENSORFLOW_ZIP, TensorFlowSavedBinarizer),
        (BinarizationType.TENSORFLOW_ZIP.value, TensorFlowSavedBinarizer),
        (BinarizationType.TORCH, TorchBinarizer),
        (BinarizationType.TORCH.value, TorchBinarizer),
        (BinarizationType.TORCH_JIT, TorchJITBinarizer),
        (BinarizationType.TORCH_JIT.value, TorchJITBinarizer),
        (BinarizationType.ONNX_INFERENCE_SESSION, InferenceSessionBinarizer),
        (BinarizationType.ONNX_INFERENCE_SESSION.value, InferenceSessionBinarizer),
        (BinarizationType.ONNX_INFERENCE_SESSION_FULL_RETURN, InferenceSessionFullReturnBinarizer),
        (BinarizationType.ONNX_INFERENCE_SESSION_FULL_RETURN.value, InferenceSessionFullReturnBinarizer),
        ('mlup.constants.BinarizationType', BinarizationType),
    ]
)
def test_get_binarizer(binarization_type, binarizer):
    if binarizer is None:
        return
    assert binarizer == get_class_by_path(binarization_type)


def test_get_binarizer_bad_binarization_type():
    try:
        get_class_by_path('not exists binarization type')
        pytest.fail('Not raised KeyError with not exists type')
    except ModuleNotFoundError as e:
        assert str(e) == str("No module named 'not exists binarization type'")

tests/unit_tests/ml/test_binarization.py
nxexox/pymlup
import logging

import numpy as np
import pandas as pd
import pytest
from pandas.testing import assert_frame_equal

from mlup.constants import ModelDataTransformerType
from mlup.ml.data_transformers.src_data_transformer import SrcDataTransformer
from mlup.utils.interspection import get_class_by_path


logger = logging.getLogger('mlup.test')

try:
    import numpy
    from mlup.ml.data_transformers.numpy_data_transformer import NumpyDataTransformer
except (ImportError, AttributeError) as e:
    logger.info(f'Numpy not installed. Skip tests. {e}')
    numpy, NumpyDataTransformer = None, None

try:
    import pandas
    from mlup.ml.data_transformers.pandas_data_transformer import PandasDataFrameTransformer
except (ImportError, AttributeError) as e:
    logger.info(f'Pandas not installed. Skip tests. {e}')
    pandas, PandasDataFrameTransformer = None, None

try:
    import tensorflow
    from mlup.ml.data_transformers.tf_tensor_data_transformer import TFTensorDataTransformer
    assert_tf_tensors = tensorflow.test.TestCase().assertAllEqual
except (ImportError, AttributeError) as e:
    logger.info(f'tensorflow not installed. Skip tests. {e}')
    tensorflow, TFTensorDataTransformer = None, None

    def assert_tf_tensors(a, b, msg):
        assert False

try:
    import torch
    from mlup.ml.data_transformers.torch_tensor_data_transformer import TorchTensorDataTransformer
    is_equal_torch_tensors = torch.equal
except (ImportError, AttributeError) as e:
    logger.info(f'PyTorch not installed. Skip tests. {e}')
    torch, TorchTensorDataTransformer = None, None

    def is_equal_torch_tensors(a, b):
        return False


@pytest.mark.parametrize(
    'data_type, expected_class', [
        (ModelDataTransformerType.SRC_TYPES, SrcDataTransformer),
        (ModelDataTransformerType.PANDAS_DF, PandasDataFrameTransformer),
        (ModelDataTransformerType.NUMPY_ARR, NumpyDataTransformer),
        (ModelDataTransformerType.TENSORFLOW_TENSOR, TFTensorDataTransformer),
        (ModelDataTransformerType.TORCH_TENSOR, TorchTensorDataTransformer),
    ]
)
def test_get_data_transformer_by_type(data_type, expected_class):
    if expected_class is None:
        return
    assert get_class_by_path(data_type) == expected_class


def test_get_data_transformer_by_type_bad_type():
    try:
        get_class_by_path('not exists transformer type')
        pytest.fail('Not raised KeyError with not exists type')
    except ModuleNotFoundError as e:
        assert str(e) == "No module named 'not exists transformer type'"


def test_get_data_transformer_by_custom_type():
    data = get_class_by_path('mlup.constants.ModelDataTransformerType')
    assert issubclass(data, ModelDataTransformerType)


class TestSrcDataTransformer:
    transformer_class = SrcDataTransformer

    def test_transform_to_model_format_from_list(self):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        pred_d = self.transformer_class().transform_to_model_format(data)
        assert pred_d == data

    def test_transform_to_model_format_from_dict_with_columns(self):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'col4': 41, 'col5': 51},
            {'col1': 12, 'col2': 22, 'col3': 32, 'col4': 42, 'col5': 52},
            {'col1': 13, 'col2': 23, 'col3': 33, 'col4': 43, 'col5': 53},
            {'col1': 14, 'col2': 24, 'col3': 34, 'col4': 44, 'col5': 54},
            {'col1': 15, 'col2': 25, 'col3': 35, 'col4': 45, 'col5': 55},
        ]
        cols = [
            {'name': 'col1', 'type': 'int'},
            {'name': 'col2', 'type': 'int'},
            {'name': 'col3', 'type': 'int'},
            {'name': 'col4', 'type': 'int'},
            {'name': 'col5', 'type': 'int'},
        ]
        # Check create
        pred_d = self.transformer_class().transform_to_model_format(data, columns=cols)
        assert pred_d == data

        # Check order by columns
        pred_d = self.transformer_class().transform_to_model_format(data, columns=cols[::-1])
        assert pred_d == data

    def test_transform_to_model_format_from_dict_without_columns(self):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'col4': 41, 'col5': 51},
            {'col1': 12, 'col2': 22, 'col3': 32, 'col4': 42, 'col5': 52},
            {'col1': 13, 'col2': 23, 'col3': 33, 'col4': 43, 'col5': 53},
            {'col1': 14, 'col2': 24, 'col3': 34, 'col4': 44, 'col5': 54},
            {'col1': 15, 'col2': 25, 'col3': 35, 'col4': 45, 'col5': 55},
        ]
        # Check create
        pred_d = self.transformer_class().transform_to_model_format(data)
        assert pred_d == data

        # Check order by first item
        reversed_data = data.copy()
        reversed_data[0] = {k: v for k, v in reversed(list(reversed_data[0].items()))}
        pred_d = self.transformer_class().transform_to_model_format(reversed_data)
        assert pred_d == reversed_data

    def test_transform_to_json_format(self):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        pred_d = self.transformer_class().transform_to_json_format(data)
        assert pred_d == data


@pytest.mark.skipif(PandasDataFrameTransformer is None, reason='pandas library not installed.')
class TestPandasDataFrameTransformer:
    transformer_class = PandasDataFrameTransformer

    def test_transform_to_model_format_from_dict(self):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'colstr': 'colstr', 'colbool': True},
            {'col1': 12, 'col2': 22, 'col3': 32, 'colstr': 'colstr', 'colbool': True},
            {'col1': 13, 'col2': 23, 'col3': 33, 'colstr': 'colstr', 'colbool': False},
            {'col1': 14, 'col2': 24, 'col3': 34, 'colstr': 'colstr', 'colbool': True},
            {'col1': 15, 'col2': 25, 'col3': 35, 'colstr': 'colstr', 'colbool': True},
        ]
        df = self.transformer_class().transform_to_model_format(data)

        assert_frame_equal(df, pd.DataFrame(data=data))
        assert df.columns.tolist() == list(data[0].keys())

    def test_transform_to_model_format_from_list(self):
        cols = [
            {'name': 'col1', 'type': 'int'},
            {'name': 'col2', 'type': 'int'},
            {'name': 'col3', 'type': 'int'},
            {'name': 'colstr', 'type': 'str'},
            {'name': 'colbool', 'type': 'bool'},
        ]
        data = [
            [11, 21, 31, 'colstr', True],
            [12, 22, 32, 'colstr', True],
            [13, 23, 33, 'colstr', False],
            [14, 24, 34, 'colstr', True],
            [15, 25, 35, 'colstr', True],
        ]
        df = self.transformer_class().transform_to_model_format(data, columns=cols)

        data_for_df = [
            {c['name']: v for c, v in zip(cols, d)}
            for d in data
        ]
        assert_frame_equal(df, pd.DataFrame(data=data_for_df))
        assert df.columns.tolist() == [c['name'] for c in cols]

    def test_transform_to_json_format(self):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'colstr': 'colstr', 'colbool': True},
            {'col1': 12, 'col2': 22, 'col3': 32, 'colstr': 'colstr', 'colbool': True},
            {'col1': 13, 'col2': 23, 'col3': 33, 'colstr': 'colstr', 'colbool': False},
            {'col1': 14, 'col2': 24, 'col3': 34, 'colstr': 'colstr', 'colbool': True},
            {'col1': 15, 'col2': 25, 'col3': 35, 'colstr': 'colstr', 'colbool': True},
        ]
        df = pd.DataFrame(data=data)
        trans_data = self.transformer_class().transform_to_json_format(df)

        assert trans_data == data

    def test_transform_to_json_format_from_list(self):
        data = [
            [{'col1': 11, 'col2': 21, 'col3': 31, 'colstr': 'colstr', 'colbool': True}],
            [{'col1': 12, 'col2': 22, 'col3': 32, 'colstr': 'colstr', 'colbool': True}],
            [{'col1': 13, 'col2': 23, 'col3': 33, 'colstr': 'colstr', 'colbool': False}],
            [{'col1': 14, 'col2': 24, 'col3': 34, 'colstr': 'colstr', 'colbool': True}],
            [{'col1': 15, 'col2': 25, 'col3': 35, 'colstr': 'colstr', 'colbool': True}],
        ]
        dfs = [pd.DataFrame(data=d) for d in data]
        trans_data = self.transformer_class().transform_to_json_format(dfs)

        assert trans_data == data

    @pytest.mark.parametrize(
        'dtype_name, dtype',
        [
            ('Float32Dtype', pandas.Float32Dtype),
            ('Float64Dtype', pandas.Float64Dtype),
            ('Int8Dtype', pandas.Int8Dtype),
            ('Int16Dtype', pandas.Int16Dtype),
            ('Int32Dtype', pandas.Int32Dtype),
            ('Int64Dtype', pandas.Int64Dtype),
            ('StringDtype', pandas.StringDtype),
            ('BooleanDtype', pandas.BooleanDtype),
        ]
    )
    def test_transform_with_different_dtype(self, dtype_name: str, dtype):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'colbool': False},
            {'col1': 12, 'col2': 22, 'col3': 32, 'colbool': False},
            {'col1': 13, 'col2': 23, 'col3': 33, 'colbool': False},
            {'col1': 14, 'col2': 24, 'col3': 34, 'colbool': True},
            {'col1': 15, 'col2': 25, 'col3': 35, 'colbool': True},
        ]
        if dtype_name == 'BooleanDtype':
            data = [{k: bool(v) for k, v in d.items()} for d in data]
        trans_data = self.transformer_class(dtype_name=dtype_name).transform_to_model_format(data)
        assert_frame_equal(trans_data, pd.DataFrame(data=data, dtype=dtype()))


@pytest.mark.skipif(NumpyDataTransformer is None, reason='numpy library not installed.')
class TestNumpyDataFrameTransformer:
    transformer_class = NumpyDataTransformer

    def test_transform_to_model_format_from_list(self):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        pred_d = self.transformer_class().transform_to_model_format(data)
        assert np.array_equal(pred_d, np.array(data))

    def test_transform_to_model_format_from_dict_with_columns(self):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'col4': 41, 'col5': 51},
            {'col1': 12, 'col2': 22, 'col3': 32, 'col4': 42, 'col5': 52},
            {'col1': 13, 'col2': 23, 'col3': 33, 'col4': 43, 'col5': 53},
            {'col1': 14, 'col2': 24, 'col3': 34, 'col4': 44, 'col5': 54},
            {'col1': 15, 'col2': 25, 'col3': 35, 'col4': 45, 'col5': 55},
        ]
        cols = [
            {'name': 'col1', 'type': 'int'},
            {'name': 'col2', 'type': 'int'},
            {'name': 'col3', 'type': 'int'},
            {'name': 'col4', 'type': 'int'},
            {'name': 'col5', 'type': 'int'},
        ]
        # Check create
        pred_d = self.transformer_class().transform_to_model_format(data, columns=cols)
        assert np.array_equal(pred_d, np.array([list(v.values()) for v in data]))

        # Check order by columns
        pred_d = self.transformer_class().transform_to_model_format(data, columns=cols[::-1])
        assert np.array_equal(pred_d, np.array([list(v.values())[::-1] for v in data]))

    def test_transform_to_model_format_from_dict_without_columns(self):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'col4': 41, 'col5': 51},
            {'col1': 12, 'col2': 22, 'col3': 32, 'col4': 42, 'col5': 52},
            {'col1': 13, 'col2': 23, 'col3': 33, 'col4': 43, 'col5': 53},
            {'col1': 14, 'col2': 24, 'col3': 34, 'col4': 44, 'col5': 54},
            {'col1': 15, 'col2': 25, 'col3': 35, 'col4': 45, 'col5': 55},
        ]
        # Check create
        pred_d = self.transformer_class().transform_to_model_format(data)
        assert np.array_equal(pred_d, np.array([list(v.values()) for v in data]))

        # Check order by first item
        reversed_data = data.copy()
        reversed_data[0] = {k: v for k, v in reversed(list(reversed_data[0].items()))}
        pred_d = self.transformer_class().transform_to_model_format(reversed_data)
        assert np.array_equal(pred_d, np.array([list(v.values())[::-1] for v in data]))

    def test_transform_to_json_format(self):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        arr = np.array(data)

        pred_d = self.transformer_class().transform_to_json_format(arr)
        assert pred_d == data

    def test_transform_to_json_format_from_list(self):
        data = [
            [[11, 21, 31, 41, 51]],
            [[12, 22, 32, 42, 52]],
            [[13, 23, 33, 43, 53]],
            [[14, 24, 34, 44, 54]],
            [[15, 25, 35, 45, 55]],
        ]
        arrs = [np.array(d) for d in data]

        pred_d = self.transformer_class().transform_to_json_format(arrs)
        assert pred_d == data

    @pytest.mark.parametrize(
        'dtype_name, dtype',
        [
            ('float64', numpy.float64),
            ('float32', numpy.float32),
            ('float16', numpy.float16),
            ('int64', numpy.int64),
            ('int32', numpy.int32),
            ('int16', numpy.int16),
            ('bool_', numpy.bool_),
        ]
    )
    def test_transform_with_different_dtype(self, dtype_name: str, dtype: numpy.generic):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        pred_d = self.transformer_class(dtype_name=dtype_name).transform_to_model_format(data)
        assert np.array_equal(pred_d, np.array(data, dtype=dtype))


@pytest.mark.skipif(TFTensorDataTransformer is None, reason='tensorflow library not installed.')
class TestTensorFlowTensorDataTransformer:
    transformer_class = TFTensorDataTransformer

    def test_transform_to_model_format_from_list(self):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        pred_d = self.transformer_class().transform_to_model_format(data)
        assert_tf_tensors(pred_d, tensorflow.convert_to_tensor(data))

    def test_transform_to_model_format_from_dict_with_columns(self):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'col4': 41, 'col5': 51},
            {'col1': 12, 'col2': 22, 'col3': 32, 'col4': 42, 'col5': 52},
            {'col1': 13, 'col2': 23, 'col3': 33, 'col4': 43, 'col5': 53},
            {'col1': 14, 'col2': 24, 'col3': 34, 'col4': 44, 'col5': 54},
            {'col1': 15, 'col2': 25, 'col3': 35, 'col4': 45, 'col5': 55},
        ]
        cols = [
            {'name': 'col1', 'type': 'int'},
            {'name': 'col2', 'type': 'int'},
            {'name': 'col3', 'type': 'int'},
            {'name': 'col4', 'type': 'int'},
            {'name': 'col5', 'type': 'int'},
        ]
        # Check create
        pred_d = self.transformer_class().transform_to_model_format(data, columns=cols)
        assert_tf_tensors(pred_d, tensorflow.convert_to_tensor([list(v.values()) for v in data]))

        # Check order by columns
        pred_d = self.transformer_class().transform_to_model_format(data, columns=cols[::-1])
        assert_tf_tensors(pred_d, tensorflow.convert_to_tensor([list(v.values())[::-1] for v in data]))

    def test_transform_to_model_format_from_dict_without_columns(self):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'col4': 41, 'col5': 51},
            {'col1': 12, 'col2': 22, 'col3': 32, 'col4': 42, 'col5': 52},
            {'col1': 13, 'col2': 23, 'col3': 33, 'col4': 43, 'col5': 53},
            {'col1': 14, 'col2': 24, 'col3': 34, 'col4': 44, 'col5': 54},
            {'col1': 15, 'col2': 25, 'col3': 35, 'col4': 45, 'col5': 55},
        ]
        # Check create
        pred_d = self.transformer_class().transform_to_model_format(data)
        assert_tf_tensors(pred_d, tensorflow.convert_to_tensor([list(v.values()) for v in data]))

        # Check order by first item
        reversed_data = data.copy()
        reversed_data[0] = {k: v for k, v in reversed(list(reversed_data[0].items()))}
        pred_d = self.transformer_class().transform_to_model_format(reversed_data)
        assert_tf_tensors(pred_d, tensorflow.convert_to_tensor([list(v.values())[::-1] for v in data]))

    def test_transform_to_json_format(self):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        arr = tensorflow.convert_to_tensor(data)

        pred_d = self.transformer_class().transform_to_json_format(arr)
        assert pred_d == data

    def test_transform_to_json_format_from_list(self):
        data = [
            [[11, 21, 31, 41, 51]],
            [[12, 22, 32, 42, 52]],
            [[13, 23, 33, 43, 53]],
            [[14, 24, 34, 44, 54]],
            [[15, 25, 35, 45, 55]],
        ]
        arrs = [tensorflow.convert_to_tensor(d) for d in data]

        pred_d = self.transformer_class().transform_to_json_format(arrs)
        assert pred_d == data

    @pytest.mark.parametrize(
        'dtype_name, dtype',
        [
            ('float16', tensorflow.float16),
            ('float32', tensorflow.float32),
            ('float64', tensorflow.float64),
            ('int16', tensorflow.int16),
            ('int32', tensorflow.int32),
            ('int64', tensorflow.int64),
            ('bool', tensorflow.bool),
        ]
    )
    def test_transform_with_different_dtype(self, dtype_name: str, dtype: tensorflow.DType):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        if dtype_name == 'bool':
            data = [[bool(i) for i in r] for r in data]
        pred_d = self.transformer_class(dtype_name=dtype_name).transform_to_model_format(data)
        assert_tf_tensors(pred_d, tensorflow.convert_to_tensor(data, dtype=dtype))


@pytest.mark.skipif(TorchTensorDataTransformer is None, reason='pytorch library not installed.')
class TestTorchTensorDataTransformer:
    transformer_class = TorchTensorDataTransformer

    def test_transform_to_model_format_from_list(self):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        pred_d = self.transformer_class().transform_to_model_format(data)
        assert is_equal_torch_tensors(pred_d, torch.tensor(data))

    def test_transform_to_model_format_from_dict_with_columns(self):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'col4': 41, 'col5': 51},
            {'col1': 12, 'col2': 22, 'col3': 32, 'col4': 42, 'col5': 52},
            {'col1': 13, 'col2': 23, 'col3': 33, 'col4': 43, 'col5': 53},
            {'col1': 14, 'col2': 24, 'col3': 34, 'col4': 44, 'col5': 54},
            {'col1': 15, 'col2': 25, 'col3': 35, 'col4': 45, 'col5': 55},
        ]
        cols = [
            {'name': 'col1', 'type': 'int'},
            {'name': 'col2', 'type': 'int'},
            {'name': 'col3', 'type': 'int'},
            {'name': 'col4', 'type': 'int'},
            {'name': 'col5', 'type': 'int'},
        ]
        # Check create
        pred_d = self.transformer_class().transform_to_model_format(data, columns=cols)
        assert is_equal_torch_tensors(pred_d, torch.tensor([list(v.values()) for v in data]))

        # Check order by columns
        pred_d = self.transformer_class().transform_to_model_format(data, columns=cols[::-1])
        assert is_equal_torch_tensors(pred_d, torch.tensor([list(v.values())[::-1] for v in data]))

    def test_transform_to_model_format_from_dict_without_columns(self):
        data = [
            {'col1': 11, 'col2': 21, 'col3': 31, 'col4': 41, 'col5': 51},
            {'col1': 12, 'col2': 22, 'col3': 32, 'col4': 42, 'col5': 52},
            {'col1': 13, 'col2': 23, 'col3': 33, 'col4': 43, 'col5': 53},
            {'col1': 14, 'col2': 24, 'col3': 34, 'col4': 44, 'col5': 54},
            {'col1': 15, 'col2': 25, 'col3': 35, 'col4': 45, 'col5': 55},
        ]
        # Check create
        pred_d = self.transformer_class().transform_to_model_format(data)
        assert is_equal_torch_tensors(pred_d, torch.tensor([list(v.values()) for v in data]))

        # Check order by first item
        reversed_data = data.copy()
        reversed_data[0] = {k: v for k, v in reversed(list(reversed_data[0].items()))}
        pred_d = self.transformer_class().transform_to_model_format(reversed_data)
        assert is_equal_torch_tensors(pred_d, torch.tensor([list(v.values())[::-1] for v in data]))

    def test_transform_to_json_format(self):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        arr = torch.tensor(data)

        pred_d = self.transformer_class().transform_to_json_format(arr)
        assert pred_d == data

    def test_transform_to_json_format_from_list(self):
        data = [
            [[11, 21, 31, 41, 51]],
            [[12, 22, 32, 42, 52]],
            [[13, 23, 33, 43, 53]],
            [[14, 24, 34, 44, 54]],
            [[15, 25, 35, 45, 55]],
        ]
        arrs = [torch.tensor(d) for d in data]

        pred_d = self.transformer_class().transform_to_json_format(arrs)
        assert pred_d == data

    @pytest.mark.parametrize(
        'dtype_name, dtype',
        [
            ('float16', torch.float16),
            ('float32', torch.float32),
            ('float64', torch.float64),
            ('int16', torch.int16),
            ('int32', torch.int32),
            ('int64', torch.int64),
            ('bool', torch.bool),
        ]
    )
    def test_transform_with_different_dtype(self, dtype_name: str, dtype: torch.dtype):
        data = [
            [11, 21, 31, 41, 51],
            [12, 22, 32, 42, 52],
            [13, 23, 33, 43, 53],
            [14, 24, 34, 44, 54],
            [15, 25, 35, 45, 55],
        ]
        if dtype_name == 'bool':
            data = [[bool(i) for i in r] for r in data]
        pred_d = self.transformer_class(dtype_name=dtype_name).transform_to_model_format(data)
        is_equal_torch_tensors(pred_d, torch.tensor(data, dtype=dtype))

tests/unit_tests/ml/test_data_transformers.py
nxexox/pymlup
import asyncio
import logging
import pickle
import threading
import time
from typing import List, Dict

import numpy as np
import pytest

from mlup.constants import ModelDataTransformerType, DEFAULT_X_ARG_NAME, StorageType, BinarizationType
from mlup.errors import ModelLoadError, PredictTransformDataError
from mlup.ml.model import MLupModel, ModelConfig
from mlup.utils.loop import create_async_task

logger = logging.getLogger('mlup.test')


class ModelWithX:
    def predict(self, X: List):
        return X


@pytest.fixture(scope="session")
def pickle_with_x_model(tmp_path_factory):
    model = ModelWithX()
    f_path = tmp_path_factory.getbasetemp()
    model_name = 'pickle_easy_model_for_test_model.pckl'
    logger.info(f'Create {f_path}/{model_name} fixture.')
    with open(f_path / model_name, 'wb') as f:
        pickle.dump(model, f)
    return f_path / model_name


class TestMLupModelPublicMethods:
    @pytest.mark.asyncio
    async def test_no_load_model(self):
        mlup_model = MLupModel(conf=ModelConfig(storage_type=StorageType.disk))
        try:
            await mlup_model.predict(X=[[1, 2, 3]])
            pytest.fail('Not raised ModelLoadError in not loaded model.')
        except ModelLoadError as e:
            assert str(e) == 'Model object not loaded. Please call load().'

    def test_not_exists_predict_method(self, pickle_with_x_model):
        ml_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_with_x_model},
                predict_method_name='not_exists_method'
            )
        )
        try:
            ml_model.load()
            pytest.fail('Not raised ModelLoadError, if not exists predict method.')
        except ModelLoadError as e:
            assert str(e) == "'ModelWithX' object has no attribute 'not_exists_method'"

    @pytest.mark.asyncio
    async def test_load_model_from_pickle_file(self, pickle_with_x_model):
        mlup_model = MLupModel(
            conf=ModelConfig(
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_with_x_model},
            )
        )
        mlup_model.load()
        await mlup_model.predict(X=[[1, 2, 3]])

    @pytest.mark.asyncio
    async def test_load_model_from_obj(self):
        ml_model = ModelWithX()
        # With default stprage_type=memory
        mlup_model = MLupModel(
            ml_model=ml_model,
            conf=ModelConfig(
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            )
        )
        mlup_model.load()
        await mlup_model.predict(X=[[1, 2, 3]])

        # With storage_type=memory
        mlup_model = MLupModel(
            ml_model=ml_model,
            conf=ModelConfig(
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
                storage_type=StorageType.memory,
            )
        )
        mlup_model.load()
        await mlup_model.predict(X=[[1, 2, 3]])

    @pytest.mark.asyncio
    async def test_load_model_from_obj_ignore_binarizer(self):
        model = ModelWithX()
        mlup_model = MLupModel(
            ml_model=model,
            conf=ModelConfig(
                storage_type=StorageType.memory,
                binarization_type=BinarizationType.JOBLIB,
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            )
        )
        mlup_model.load()
        assert model == mlup_model.model_obj
        pred_res = await mlup_model.predict(X=[[1, 2, 3]])
        assert pred_res == [[1, 2, 3]]

    @pytest.mark.parametrize(
        'force_loading, expected, default',
        [
            pytest.param(False, 'test', None, id='force_loading=False'),
            pytest.param(True, 'attr not found', 'attr not found', id='force_loading=True')
        ]
    )
    def test_loading_binary_model_from_pickle_with_force_loading(
        self, pickle_print_model, pickle_print_sleep_model, force_loading: bool, expected: str, default
    ):
        mlup_model = MLupModel(
            conf=ModelConfig(
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model}
            )
        )
        mlup_model.load()

        # Set attribute
        mlup_model.model_obj.TestAttribute = 'test'
        assert getattr(mlup_model.model_obj, 'TestAttribute') == 'test'

        # Reloading model
        mlup_model.conf.storage_kwargs['path_to_files'] = pickle_print_sleep_model
        try:
            mlup_model.load(force_loading=force_loading)
        except ModelLoadError as e:
            assert str(e) == 'Model is already loaded. For reload use force_reloading=True.'
            assert force_loading is False
        assert getattr(mlup_model.model_obj, 'TestAttribute', default) == expected

    @pytest.mark.parametrize(
        'force_loading, expected, default',
        [
            pytest.param(False, 'test', None, id='force_loading=False'),
            pytest.param(True, 'attr not found', 'attr not found', id='force_loading=True')
        ]
    )
    def test_loading_binary_model_from_obj_with_force_loading(
        self, force_loading: bool, expected: str, default
    ):
        ml_model = ModelWithX()
        mlup_model = MLupModel(
            ml_model=ml_model,
            conf=ModelConfig(
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            )
        )
        mlup_model.load()

        # Set attribute
        mlup_model.model_obj.TestAttribute = 'test'
        assert getattr(mlup_model.model_obj, 'TestAttribute') == 'test'

        # Reloading my model
        ml_model = ModelWithX()
        mlup_model.conf.storage_kwargs['model'] = ml_model
        try:
            mlup_model.load(force_loading=force_loading)
        except ModelLoadError as e:
            assert str(e) == 'Model is already loaded. For reload use force_reloading=True.'
            assert force_loading is False
        assert getattr(mlup_model.model_obj, 'TestAttribute', default) == expected

    def test_load_model_error(self):
        try:
            MLupModel()
            pytest.fail('Not raised ModelLoadError, if not set path to model.')
        except ModelLoadError as e:
            assert str(e) == f'If you use {StorageType.memory}, need set "ml" argument.'

    def test_load_model_from_pickle_not_exist_file(self, pickle_not_exists_file):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_not_exists_file},
            )
        )
        try:
            mlup_model.load()
            pytest.fail('Not raised ModelLoadError, if not exists file.')
        except ModelLoadError as e:
            assert str(e).startswith('[Errno 2] No such file or directory: ')

    def test_load_model_from_pickle_bad_reload(self, pickle_print_model, pickle_print_sleep_model):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
            )
        )
        mlup_model.load()
        mlup_model.conf.storage_kwargs['path_to_files'] = pickle_print_sleep_model
        try:
            mlup_model.load(force_loading=False)
            pytest.fail('Not raise ModelLoadError, because not use force_loading=True')
        except ModelLoadError as e:
            assert str(e) == 'Model is already loaded. For reload use force_reloading=True.'

    def test_reload_binary_model_from_pickle_change_path(self, pickle_print_model, pickle_print_sleep_model):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model}
            )
        )
        mlup_model.load()

        # Check not reload
        setattr(mlup_model.model_obj, 'test', 'test')
        mlup_model.load(force_loading=True)
        assert hasattr(mlup_model.model_obj, 'test') is False
        src_model = mlup_model.model_obj

        # Check reload model
        mlup_model.conf.storage_kwargs['path_to_files'] = pickle_print_sleep_model
        mlup_model.load(force_loading=True)
        assert hasattr(mlup_model.model_obj, 'test') is False
        assert id(src_model) != id(mlup_model.model_obj)

    def test_load_model_from_pickle_change_predict_method(self, pickle_print_model, pickle_print_sleep_model):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model}
            )
        )
        mlup_model.load()
        predict_method = mlup_model.model_obj.predict
        assert mlup_model._predict_method == predict_method

        # Check not reload
        mlup_model.conf.storage_kwargs['path_to_files'] = pickle_print_sleep_model
        try:
            mlup_model.load()
            pytest.fail('Not raise ModelLoadError, because not use force_loading=True')
        except ModelLoadError:
            assert mlup_model._predict_method == predict_method

        # Check reload model
        mlup_model.load(force_loading=True)
        assert mlup_model._predict_method != predict_method
        assert mlup_model._predict_method == mlup_model.model_obj.predict

        # Check change predict_method_name
        predict_method = mlup_model._predict_method
        mlup_model.conf.predict_method_name = 'second_predict'
        mlup_model.load(force_loading=True)
        assert mlup_model._predict_method != predict_method

    def test_load_model_from_pickle_change_x_column_name(self, pickle_print_model, pickle_print_sleep_model):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
            )
        )
        mlup_model.load()
        x_column_name = mlup_model.x_column_name
        assert mlup_model.x_column_name == x_column_name

        # Check not reload
        try:
            mlup_model.load()
            pytest.fail('Not raise ModelLoadError, because not use force_loading=True')
        except ModelLoadError:
            assert mlup_model.x_column_name == x_column_name

        # Check reload model
        mlup_model.conf.storage_kwargs['path_to_files'] = pickle_print_sleep_model
        mlup_model.load(force_loading=True)
        assert mlup_model.x_column_name == x_column_name

        # Check change predict_method_name
        x_column_name = mlup_model.x_column_name
        mlup_model.conf.predict_method_name = 'predict_with_x_name_y'
        mlup_model.load(force_loading=True)
        assert mlup_model.x_column_name != x_column_name
        assert mlup_model.x_column_name == 'Y'

    def test_load_model_from_pickle_change_predict_arguments(
        self, pickle_with_x_model, pickle_print_sleep_model
    ):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_with_x_model},
            )
        )
        mlup_model.load()
        predict_arguments = mlup_model._predict_arguments
        assert mlup_model._predict_arguments == predict_arguments

        # Check not reload
        try:
            mlup_model.load()
            pytest.fail('Not raise ModelLoadError, because not use force_loading=True')
        except ModelLoadError:
            assert mlup_model._predict_arguments == predict_arguments

        # Check reload model
        mlup_model.conf.storage_kwargs['path_to_files'] = pickle_print_sleep_model
        mlup_model.load(force_loading=True)
        assert mlup_model._predict_arguments != predict_arguments

        # Check change predict_method_name
        predict_arguments = mlup_model._predict_arguments
        mlup_model.conf.predict_method_name = 'predict_with_x_name_y'
        mlup_model.load(force_loading=True)
        assert mlup_model._predict_arguments != predict_arguments
        assert mlup_model._predict_arguments == [
            {'name': 'Y', 'required': True, 'type': 'List', 'is_X': True},
            {'name': 'test_param', 'required': False, 'type': 'bool', 'default': False},
        ]

    def test_load_model_from_pickle_change_use_thread_loop(self, pickle_print_model, pickle_print_sleep_model):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
            )
        )
        mlup_model.load()
        assert mlup_model.conf.use_thread_loop is True
        assert mlup_model._pool_workers is not None

        # Check not reload
        try:
            mlup_model.load()
            pytest.fail('Not raise ModelLoadError, because not use force_loading=True')
        except ModelLoadError:
            assert mlup_model.conf.use_thread_loop is True
            assert mlup_model._pool_workers is not None

        # Check reload model
        mlup_model.conf.storage_kwargs['path_to_files'] = pickle_print_sleep_model
        mlup_model.load(force_loading=True)
        assert mlup_model.conf.use_thread_loop is True
        assert mlup_model._pool_workers is not None

        # Check change use_thread_loop
        mlup_model.conf.use_thread_loop = False
        mlup_model.load(force_loading=True)
        assert mlup_model.conf.use_thread_loop is False
        assert mlup_model._pool_workers is None

    def test_load_model_settings_without_model(self):
        mlup_model = MLupModel(conf=ModelConfig(storage_type=StorageType.disk))
        try:
            mlup_model.load_model_settings()
            pytest.fail('Not raised ModelLoadError in not loaded model.')
        except ModelLoadError as e:
            assert str(e) == 'Model not loaded to memory. Analyze impossible. Please call ml.load().'

    @pytest.mark.parametrize(
        'kwargs',
        [
            pytest.param({}, id='default_values'),
            pytest.param(dict(predict_method_name='predict'), id='not_default_values')
        ]
    )
    def test_load_model_settings_change_predict_method_name(self, pickle_print_model, kwargs: Dict):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
                **kwargs,
            )
        )
        mlup_model.load()
        src_predict_method = mlup_model._predict_method

        mlup_model.conf.predict_method_name = 'second_predict'
        mlup_model.load_model_settings()
        assert src_predict_method != mlup_model._predict_method

    @pytest.mark.parametrize(
        'kwargs',
        [
            pytest.param({}, id='default_values'),
            pytest.param(dict(predict_method_name='predict'), id='not_default_values')
        ]
    )
    def test_load_model_settings_change_predict_arguments(self, pickle_print_model, kwargs: Dict):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
                **kwargs,
            )
        )
        mlup_model.load()
        src_predict_arguments = mlup_model._predict_arguments

        mlup_model.conf.predict_method_name = 'predict_with_x_name_y'
        mlup_model.load_model_settings()
        assert src_predict_arguments != mlup_model._predict_arguments

    @pytest.mark.parametrize(
        'kwargs',
        [
            pytest.param({}, id='default_values'),
            pytest.param(
                dict(predict_method_name='predict', auto_detect_predict_params=True),
                id='not_default_values'
            )
        ]
    )
    def test_load_model_settings_change_x_column_name(self, pickle_print_model, kwargs: Dict):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
                **kwargs,
            )
        )
        mlup_model.load()
        src_x_column_name = mlup_model.x_column_name

        # Check if change method, to method with other signature
        mlup_model.conf.predict_method_name = 'predict_with_x_name_y'
        mlup_model.load_model_settings()
        assert src_x_column_name != mlup_model.x_column_name

        new_x_column_name = mlup_model.x_column_name
        # Check if set auto_detect_predict_params=False
        mlup_model.conf.auto_detect_predict_params = False
        mlup_model.load_model_settings()
        assert mlup_model.x_column_name != src_x_column_name
        assert mlup_model.x_column_name != new_x_column_name
        assert mlup_model.x_column_name == DEFAULT_X_ARG_NAME

    @pytest.mark.parametrize(
        'kwargs',
        [
            pytest.param({}, id='default_values'),
            pytest.param(dict(use_thread_loop=True), id='not_default_values')
        ]
    )
    def test_load_model_settings_change_use_thread_loop(self, pickle_print_model, kwargs: Dict):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
                **kwargs,
            )
        )
        mlup_model.load()
        src_pool_workers = mlup_model._pool_workers

        mlup_model.conf.use_thread_loop = False
        mlup_model.load_model_settings()
        assert mlup_model._pool_workers is None
        assert src_pool_workers != mlup_model._pool_workers

    def test_get_X_from_predict_data_not_loaded_model(self, pickle_print_model):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model}
            )
        )
        try:
            mlup_model.get_X_from_predict_data({})
            pytest.fail('Not raised ModelLoadError in not loaded model.')
        except ModelLoadError as e:
            assert str(e) == 'Model object not loaded. Please call load().'

    @pytest.mark.parametrize(
        'auto_detect_predict_params, data, expected',
        [
            pytest.param(False, {'X': [1, 2, 3]}, None, id='auto_detect_predict_params=False,key=X'),
            pytest.param(
                False, {'test': [1, 2, 3]}, None,
                id='auto_detect_predict_params=False,key=test'
            ),
            pytest.param(
                False, {DEFAULT_X_ARG_NAME: [1, 2, 3]}, [1, 2, 3],
                id='auto_detect_predict_params=False,key=DEFAULT_X_ARG_NAME'
            ),
            pytest.param(True, {'X': [1, 2, 3]}, [1, 2, 3], id='auto_detect_predict_params=True,key=X'),
            pytest.param(True, {'test': [1, 2, 3]}, None, id='auto_detect_predict_params=True,key=test'),
            pytest.param(
                True, {DEFAULT_X_ARG_NAME: [1, 2, 3]}, None,
                id='auto_detect_predict_params=True,key=DEFAULT_X_ARG_NAME'
            ),
        ]
    )
    @pytest.mark.parametrize('remove', [False, True], ids=lambda x: f'remove={x}')
    def test_get_X_from_predict_data_default_column_name(
        self, pickle_print_model, auto_detect_predict_params: bool, data: Dict, expected, remove: bool
    ):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
                auto_detect_predict_params=auto_detect_predict_params,
            )
        )
        mlup_model.load()
        pred_col = mlup_model.get_X_from_predict_data({'test': [1, 2, 3]})
        assert pred_col is None

        pred_col = mlup_model.get_X_from_predict_data(data, remove=remove)
        assert pred_col == expected
        if remove:
            assert mlup_model.x_column_name not in data

    @pytest.mark.asyncio
    async def test_predict_not_loaded_model(self, pickle_print_model):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
            )
        )
        try:
            await mlup_model.predict(X=[[1, 2, 3]])
        except ModelLoadError as e:
            assert str(e) == 'Model object not loaded. Please call load().'

    @pytest.mark.asyncio
    async def test_easy_predict(self, pickle_print_model):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            )
        )
        mlup_model.load()
        pred_result = await mlup_model.predict(X=[[1, 2, 3]])
        assert pred_result == [[1, 2, 3]]

    @pytest.mark.parametrize(
        'kwargs, data, error_msg',
        [
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.PANDAS_DF,
                 'data_transformer_for_predicted': ModelDataTransformerType.PANDAS_DF},
                [[1, 2, 3]],
                'If input data have type list, than need set columns.',
                id='LIST-TO-DF-FROM-DF'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.PANDAS_DF,
                 'data_transformer_for_predicted': ModelDataTransformerType.PANDAS_DF},
                [1, 2, 3],
                "'int' object has no attribute 'items'",
                id='PRIMITIVE-TO-DF-FROM-DF'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.NUMPY_ARR,
                 'data_transformer_for_predicted': ModelDataTransformerType.PANDAS_DF},
                [1, 2, 3],
                "'int' object has no attribute 'items'",
                id='PRIMITIVE-TO-NP-FROM-DF'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.NUMPY_ARR,
                 'data_transformer_for_predicted': ModelDataTransformerType.PANDAS_DF},
                [[1, 2, 3]],
                "'numpy.ndarray' object has no attribute 'to_dict'",
                id='PRIMITIVE-TO-NP-FROM-DF'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.PANDAS_DF,
                 'data_transformer_for_predicted': ModelDataTransformerType.NUMPY_ARR},
                [{'col1': 1, 'col2': 2, 'col3': 3}],
                "'DataFrame' object has no attribute 'tolist'",
                id='DICTS-TO-DF-FROM-NP'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.PANDAS_DF,
                 'data_transformer_for_predicted': ModelDataTransformerType.NUMPY_ARR,
                 'columns': [{'name': 'col1'}, {'name': 'col2'}, {'name': 'col3'}]},
                [[1, 2, 3]],
                "'DataFrame' object has no attribute 'tolist'",
                id='DICTS-TO-DF-FROM-NP-with_columns'
            ),
        ]
    )
    @pytest.mark.asyncio
    async def test_predict_with_not_correct_data_transformers(
        self, pickle_print_model, kwargs: Dict, data, error_msg
    ):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
                **kwargs
            )
        )
        mlup_model.load()
        try:
            await mlup_model.predict(X=data)
            pytest.fail('Not raised TransformDataError')
        except PredictTransformDataError as e:
            assert str(e) == error_msg

    @pytest.mark.parametrize(
        'kwargs, data, expected_data',
        [
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.PANDAS_DF,
                 'data_transformer_for_predicted': ModelDataTransformerType.PANDAS_DF},
                [{}],
                [],
                id='EMPTY-DICT-TO-DF-FROM-DF'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.PANDAS_DF,
                 'data_transformer_for_predicted': ModelDataTransformerType.PANDAS_DF},
                [],
                [],
                id='EMPTY-TO-DF-FROM-DF'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.PANDAS_DF,
                 'data_transformer_for_predicted': ModelDataTransformerType.PANDAS_DF,
                 'columns': [{'name': 'col1'}, {'name': 'col2'}, {'name': 'col3'}]},
                [[1, 2, 3]],
                [{'col1': 1, 'col2': 2, 'col3': 3}],
                id='LIST-TO-DF-FROM-DF-with_columns_in_construct'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.PANDAS_DF,
                 'data_transformer_for_predicted': ModelDataTransformerType.PANDAS_DF},
                [{'col1': 1, 'col2': 2, 'col3': 3}],
                [{'col1': 1, 'col2': 2, 'col3': 3}],
                id='LIST-TO-DF-FROM-DF-with_columns_in_request'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.NUMPY_ARR,
                 'data_transformer_for_predicted': ModelDataTransformerType.NUMPY_ARR},
                [[1, 2, 3]],
                [[1, 2, 3]],
                id='LIST-TO-NP-FROM-NP'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.NUMPY_ARR,
                 'data_transformer_for_predicted': ModelDataTransformerType.NUMPY_ARR},
                [],
                [],
                id='EMPTY-TO-NP-FROM-NP'
            ),
            pytest.param(
                {'data_transformer_for_predict': ModelDataTransformerType.NUMPY_ARR,
                 'data_transformer_for_predicted': ModelDataTransformerType.NUMPY_ARR},
                [[]],
                [[]],
                id='EMPTY-LIST-TO-NP-FROM-NP'
            ),
        ]
    )
    @pytest.mark.asyncio
    async def test_predict_with_correct_data_transformers(
        self, pickle_print_model, kwargs: Dict, data, expected_data
    ):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
                **kwargs
            )
        )
        mlup_model.load()
        pred_data = await mlup_model.predict(X=data)
        assert pred_data == expected_data

    @pytest.mark.asyncio
    async def test_predict_in_thread_pool_executor(self):
        class ThreadIdModel:
            def predict(self, X: List):
                return np.array([threading.current_thread().ident])
        mlup_model = MLupModel(
            ml_model=ThreadIdModel(),
            conf=ModelConfig(
                use_thread_loop=True,
                max_thread_loop_workers=5,
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            )
        )
        mlup_model.load()

        # Check, that code was launched in ThreadPoolExecutor workers.
        thread_id = await mlup_model.predict(X=[[1, 2, 3]])
        thread_pool_ids = {th.ident for th in mlup_model._pool_workers._threads}
        assert thread_id[0] in thread_pool_ids

        # Check, other thread_id, if set use_thread=False
        mlup_model.conf.use_thread_loop = False
        thread_id = await mlup_model.predict(X=[[1, 2, 3]])
        assert thread_id[0] not in thread_pool_ids

    @pytest.mark.asyncio
    async def test_predict_without_thread_pool_executor(self):
        class ThreadIdModel:
            def predict(self, X: List):
                return np.array([threading.current_thread().ident])
        mlup_model = MLupModel(
            ml_model=ThreadIdModel(),
            conf=ModelConfig(
                use_thread_loop=False,
                max_thread_loop_workers=1,
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            )
        )
        mlup_model.load()

        # Check, many predicted in single thread.
        thread_ids = {
            (await mlup_model.predict(X=[[1, 2, 3]]))[0],
            (await mlup_model.predict(X=[[1, 2, 3]]))[0],
            (await mlup_model.predict(X=[[1, 2, 3]]))[0],
            (await mlup_model.predict(X=[[1, 2, 3]]))[0],
            (await mlup_model.predict(X=[[1, 2, 3]]))[0],
            (await mlup_model.predict(X=[[1, 2, 3]]))[0],
            (await mlup_model.predict(X=[[1, 2, 3]]))[0],
        }
        assert mlup_model._pool_workers is None
        assert len(thread_ids) == 1

    @pytest.mark.parametrize(
        'use_thread_loop, max_thread_loop_workers',
        [
            pytest.param(False, 1, id='multiprocessing.Lock'),
            pytest.param(True, 1, id='asyncio.locks.Lock'),
        ]
    )
    @pytest.mark.asyncio
    async def test_predict_lock(
        self, pickle_print_model, pickle_print_sleep_model, use_thread_loop: bool, max_thread_loop_workers: int
    ):
        mlup_model = MLupModel(
            conf=ModelConfig(
                storage_type=StorageType.disk,
                storage_kwargs={'path_to_files': pickle_print_model},
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
                use_thread_loop=use_thread_loop,
                max_thread_loop_workers=max_thread_loop_workers,
            )
        )
        mlup_model.load()

        async def call_predict(mlup_model, data):
            return await mlup_model.predict(**data)

        # Check Lock not sleep model. Fast work
        tasks = []
        start = time.monotonic()
        for i in range(3):
            tasks.append(
                create_async_task(
                    call_predict(mlup_model, {'X': [[1, 2, 3]]}),
                    name=f'test_predict_lock-not_sleep_{i}',
                )
            )
        await asyncio.gather(*tasks)
        assert time.monotonic() - start < 1

        # Check Lock with sleep model. Slowly work
        mlup_model.conf.storage_kwargs['path_to_files'] = pickle_print_sleep_model
        mlup_model.load(force_loading=True)
        mlup_model.model_obj.sleep = 0.35
        tasks = []
        start = time.monotonic()
        for i in range(3):
            tasks.append(
                create_async_task(
                    call_predict(mlup_model, {'X': [[1, 2, 3]]}),
                    name=f'test_predict_lock-with_sleep_{i}'
                )
            )
        await asyncio.gather(*tasks)
        # 1 second per 1 call predict
        assert time.monotonic() - start > 1

    @pytest.mark.asyncio
    async def test_predict_from_numpy(self):
        mlup_model = MLupModel(
            ml_model=ModelWithX(),
            conf=ModelConfig(
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
                data_transformer_for_predicted=ModelDataTransformerType.SRC_TYPES,
            )
        )
        mlup_model.load()

        predicted_data = await mlup_model.predict_from(X=[1, 2, 3, 4, 5, 6, 7, 8, 9])
        np.array_equal(predicted_data, np.array([1, 2, 3, 4, 5, 6, 7, 8, 9]))

tests/unit_tests/ml/test_model.py
nxexox/pymlup
from mlup.constants import ModelLibraryType, StorageType, ModelDataTransformerType
from mlup.ml.model import ModelConfig


def test_model_config_default_values():
    conf = ModelConfig()
    assert conf.name == 'MyFirstMLupModel'
    assert conf.version == '1.0.0.0'
    assert conf.type == ModelLibraryType.SKLEARN
    assert conf.columns is None
    assert conf.predict_method_name == 'predict'
    assert conf.auto_detect_predict_params is True
    assert conf.storage_type == StorageType.memory
    assert conf.storage_kwargs == {}
    assert conf.binarization_type == 'auto'
    assert conf.use_thread_loop is True
    assert conf.max_thread_loop_workers is None
    assert conf.data_transformer_for_predict == ModelDataTransformerType.NUMPY_ARR
    assert conf.data_transformer_for_predicted == ModelDataTransformerType.NUMPY_ARR

tests/unit_tests/ml/test_model_config.py
nxexox/pymlup
import logging
import os
import pickle
import shutil

import pytest

from mlup.constants import StorageType
from mlup.ml.storage.memory import MemoryStorage
from mlup.ml.storage.local_disk import DiskStorage
from mlup.utils.interspection import get_class_by_path


logger = logging.getLogger('mlup.test')


@pytest.fixture(scope="session")
def pickle_with_x_model(model_with_x_class, tmp_path_factory):
    f_path = tmp_path_factory.getbasetemp() / 'model_with_x'
    os.makedirs(f_path, exist_ok=True)
    model_name = 'pickle_easy_model_for_test_storage.pckl'
    logger.info(f'Create {f_path}/{model_name} fixture.')
    with open(f_path / model_name, 'wb') as f:
        pickle.dump(model_with_x_class(), f)
    return f_path / model_name


@pytest.fixture(scope="session")
def pickle_with_x_model_two(model_with_x_class, tmp_path_factory):
    f_path = tmp_path_factory.getbasetemp() / 'model_with_x'
    os.makedirs(f_path, exist_ok=True)
    model_name = 'pickle_easy_model_for_test_storage_two.pckl'
    logger.info(f'Create {f_path}/{model_name} fixture.')
    with open(f_path / model_name, 'wb') as f:
        pickle.dump(model_with_x_class(), f)
    return f_path / model_name


@pytest.mark.parametrize(
    'storage_type, storage_class', [
        (StorageType.memory, MemoryStorage),
        (StorageType.memory.value, MemoryStorage),
        (StorageType.disk, DiskStorage),
        (StorageType.disk.value, DiskStorage),
        ('mlup.constants.StorageType', StorageType),
    ]
)
def test_get_storage_class(storage_type, storage_class):
    assert get_class_by_path(storage_type) == storage_class


def test_get_storage_class_bad_storage_type():
    try:
        get_class_by_path('not exists storage type')
        pytest.fail('Not raised KeyError with not exists type')
    except ModuleNotFoundError as e:
        assert str(e) == str("No module named 'not exists storage type'")


class TestMemoryStorage:
    def test_load(self, model_with_x):
        storage = MemoryStorage(model=model_with_x)
        pred_models = storage.load()
        assert len(pred_models) == 1
        assert pred_models[0].raw_data == model_with_x

    def test_load_bytes_single_file(self, model_with_x):
        storage = MemoryStorage(model=model_with_x)
        pred_model = storage.load_bytes_single_file()
        assert pred_model == model_with_x


class TestDiskStorage:
    def test_load_from_single_file(self, pickle_with_x_model):
        storage = DiskStorage(path_to_files=pickle_with_x_model, need_load_file=True)
        models_bin = storage.load()
        assert len(models_bin) == 1
        with open(pickle_with_x_model, 'rb') as f:
            src_data = f.read()
        assert str(models_bin[0].path) == str(pickle_with_x_model)
        assert models_bin[0].raw_data == src_data

    def test_load_single_from_folder(self, pickle_with_x_model, joblib_print_model):
        # joblib_print_model for second file in folder
        storage = DiskStorage(path_to_files=os.path.dirname(pickle_with_x_model), need_load_file=True)
        models_bin = storage.load()
        assert len(models_bin) == 1
        with open(pickle_with_x_model, 'rb') as f:
            src_data = f.read()
        assert str(models_bin[0].path) == str(pickle_with_x_model)
        assert models_bin[0].raw_data == src_data

    def test_load_many_from_folder(self, pickle_with_x_model, pickle_with_x_model_two):
        storage = DiskStorage(path_to_files=os.path.dirname(pickle_with_x_model), need_load_file=True)
        models_bin = storage.load()
        assert len(models_bin) == 2
        with open(pickle_with_x_model, 'rb') as f:
            with_x_model_data = f.read()
        with open(pickle_with_x_model_two, 'rb') as f:
            with_x_model_data_two = f.read()
        # Order in folder
        assert str(models_bin[0].path) == str(pickle_with_x_model)
        assert models_bin[0].raw_data == with_x_model_data
        assert str(models_bin[1].path) == str(pickle_with_x_model_two)
        assert models_bin[1].raw_data == with_x_model_data_two

    def test_load_single_file_by_not_default_mask(self, joblib_print_model):
        storage = DiskStorage(
            path_to_files=joblib_print_model,
            files_mask=r'(\w.-_)*.joblib',
            need_load_file=True,
        )
        models_bin = storage.load()
        assert len(models_bin) == 1
        with open(joblib_print_model, 'rb') as f:
            src_data = f.read()
        assert str(models_bin[0].path) == str(joblib_print_model)
        assert models_bin[0].raw_data == src_data

    def test_load_many_file_by_not_default_mask(
        self, pickle_with_x_model, pickle_print_model, joblib_print_model, joblib_print_sleep_model
    ):
        path_to_folder = os.path.join(os.path.dirname(joblib_print_model), 'test_load_many_file_by_not_default_mask')
        os.makedirs(path_to_folder)
        joblib_print_model_new_path = os.path.join(path_to_folder, os.path.basename(joblib_print_model))
        joblib_print_sleep_model_new_path = os.path.join(path_to_folder, os.path.basename(joblib_print_sleep_model))
        shutil.copyfile(joblib_print_model, joblib_print_model_new_path)
        shutil.copyfile(joblib_print_sleep_model, joblib_print_sleep_model_new_path)
        shutil.copyfile(pickle_with_x_model, os.path.join(path_to_folder, os.path.basename(pickle_with_x_model)))
        shutil.copyfile(pickle_print_model, os.path.join(path_to_folder, os.path.basename(pickle_print_model)))

        storage = DiskStorage(
            path_to_files=path_to_folder,
            files_mask=r'(\w.-_)*.joblib',
            need_load_file=True,
        )
        models_bin = storage.load()
        assert len(models_bin) == 2
        with open(joblib_print_model_new_path, 'rb') as f:
            print_model_data = f.read()
        with open(joblib_print_sleep_model_new_path, 'rb') as f:
            print_sleep_model_data = f.read()
        # Order in folder
        for m in models_bin:
            assert str(m.path) in (str(joblib_print_model_new_path), str(joblib_print_sleep_model_new_path))
            assert m.raw_data in (print_model_data, print_sleep_model_data)

    # Folder, file
    @pytest.mark.parametrize(
        'model_path',
        ['pickle_not_exists_file', 'pickle_not_exists_folder'],
        ids=['file', 'folder']
    )
    def test_load_from_not_exists_path(self, model_path, request):
        model_path = request.getfixturevalue(model_path)
        storage = DiskStorage(path_to_files=model_path, need_load_file=True)
        try:
            storage.load()
            pytest.fail('Not raise ModelLoadError.')
        except FileNotFoundError as e:
            assert str(e) == f"[Errno 2] No such file or directory: '{model_path}'"

    def test_load_bytes_single_file(self, pickle_print_model):
        storage = DiskStorage(path_to_files='', need_load_file=True)
        model_bin = storage.load_bytes_single_file(path_to_file=pickle_print_model)
        with open(pickle_print_model, 'rb') as f:
            src_data = f.read()
        assert str(model_bin.path) == str(pickle_print_model)
        assert model_bin.raw_data == src_data

    def test_load_bytes_single_file_not_exists_file(self, pickle_not_exists_file):
        storage = DiskStorage(path_to_files='', need_load_file=True)
        try:
            storage.load_bytes_single_file(path_to_file=pickle_not_exists_file)
        except FileNotFoundError as e:
            assert str(e) == f"[Errno 2] No such file or directory: '{pickle_not_exists_file}'"

tests/unit_tests/ml/test_storage.py
nxexox/pymlup
import json
import logging
from dataclasses import fields, dataclass, make_dataclass, field

import pytest
import yaml

from mlup.config import ConfigProvider, EXCLUDE_KEYS, EXCLUDE_CONDITION_KEYS, CONFIGS_VERSION
from mlup.constants import ModelLibraryType, StorageType, ModelDataTransformerType, ITEM_ID_COL_NAME, \
    WebAppArchitecture
from mlup.up import Config
from mlup.ml.model import ModelConfig
from mlup.web.app import WebAppConfig


logger = logging.getLogger('mlup.test')


_test_config = {
    'name': 'MyFirstMLupModel',
    'version': '1.0.0.0',
    'type': 'ModelLibraryType.SKLEARN',
    'columns': None,
    'custom_column_pydantic_model': None,
    'predict_method_name': 'predict',
    'auto_detect_predict_params': True,
    'storage_type': 'StorageType.memory',
    'storage_kwargs': {},
    'binarization_type': 'BinarizationType.PICKLE',
    'use_thread_loop': True,
    'max_thread_loop_workers': None,
    'data_transformer_for_predict': 'ModelDataTransformerType.PANDAS_DF',
    'data_transformer_for_predicted': 'ModelDataTransformerType.NUMPY_ARR',
    'dtype_for_predict': None,
    'host': '0.0.0.0',
    'port': 8009,
    'web_app_version': '1.0.0.0',
    'column_validation': False,
    'mode': 'WebAppArchitecture.directly_to_predict',
    'max_queue_size': 100,
    'ttl_predicted_data': 60,
    'ttl_client_wait': 30.0,
    'min_batch_len': 10,
    'batch_worker_timeout': 1.0,
    'is_long_predict': False,
    'throttling_max_requests': None,
    'throttling_max_request_len': None,
    'timeout_for_shutdown_daemon': 3.0,
    'item_id_col_name': 'ITEM_ID_COL_NAME',
    'debug': False,
    'show_docs': True,
    'uvicorn_kwargs': {},
}


def test_up_config_default_values():
    conf = Config()
    # Model configs
    assert conf.name == 'MyFirstMLupModel'
    assert conf.version == '1.0.0.0'
    assert conf.type == ModelLibraryType.SKLEARN
    assert conf.columns is None
    assert conf.predict_method_name == 'predict'
    assert conf.auto_detect_predict_params is True
    assert conf.storage_type == StorageType.memory
    assert conf.storage_kwargs == {}
    assert conf.binarization_type == 'auto'
    assert conf.use_thread_loop is True
    assert conf.max_thread_loop_workers is None
    assert conf.data_transformer_for_predict == ModelDataTransformerType.NUMPY_ARR
    assert conf.data_transformer_for_predicted == ModelDataTransformerType.NUMPY_ARR
    assert conf.dtype_for_predict is None

    # WebAPp configs
    assert conf.host == '0.0.0.0'
    assert conf.port == 8009
    assert conf.web_app_version == '1.0.0.0'
    assert conf.column_validation is False
    assert conf.custom_column_pydantic_model is None
    assert conf.mode == WebAppArchitecture.directly_to_predict
    assert conf.max_queue_size == 100
    assert conf.ttl_predicted_data == 60
    assert conf.ttl_client_wait == 30.0
    assert conf.min_batch_len == 10
    assert conf.batch_worker_timeout == 1.0
    assert conf.is_long_predict is False
    assert conf.throttling_max_requests is None
    assert conf.throttling_max_request_len is None
    assert conf.timeout_for_shutdown_daemon == 3.0
    assert conf.item_id_col_name == ITEM_ID_COL_NAME
    assert conf.debug is False
    assert conf.show_docs is True
    assert conf.uvicorn_kwargs == {}

    config_actual_fields = set()
    for n, f in Config.__dataclass_fields__.items():
        if f.repr is True:
            config_actual_fields.add(n)

    assert config_actual_fields - _test_config.keys() == set()


def test_config_have_unique_keys():
    model_conf_fields = {f.name for f in fields(ModelConfig)}
    web_app_conf_fields = {f.name for f in fields(WebAppConfig)}
    assert model_conf_fields & web_app_conf_fields == set()


@dataclass
class _Conf:
    pass


@dataclass
class ForExample:
    @dataclass
    class _ML:
        conf = _Conf()

    @dataclass
    class _WEB:
        conf = _Conf()

    ml: _ML = field(default_factory=_ML)
    web: _WEB = field(default_factory=_WEB)


class TestConfigProvider:
    def make_for_example_with_new_config(self, new_config: _Conf):
        @dataclass
        class NewForExample:
            @dataclass
            class _ML:
                conf = new_config

            @dataclass
            class _WEB:
                conf = new_config
            ml: _ML = field(default_factory=_ML)
            web: _WEB = field(default_factory=_WEB)
        return NewForExample()

    def test_get_config_dict(self):
        obj = ForExample()
        provider = ConfigProvider(obj, configs_objects_map={"ml": obj._ML.conf, "web": obj._WEB.conf})
        conf_dict = provider.get_config_dict()
        assert conf_dict == {'ml': {}, 'web': {}}

    def test_get_config_dict_with_data(self):
        @dataclass
        class _NewConf(_Conf):
            without_default_int: int
            default_string: str = 'default_string'

        obj = self.make_for_example_with_new_config(_NewConf(123))
        provider = ConfigProvider(obj, configs_objects_map={"ml": obj._ML.conf, "web": obj._WEB.conf})
        config_dict = provider.get_config_dict()
        assert config_dict == {
            'ml': {'without_default_int': 123, 'default_string': 'default_string'},
            'web': {'without_default_int': 123, 'default_string': 'default_string'}
        }

    def test_get_config_dict_with_exclude_keys(self):
        _NewConf = make_dataclass(
            '_NewConf',
            [
                (attr_name, str, field(default=f'default_{attr_name}'))
                for attr_name in set([a for b in EXCLUDE_KEYS.values() for a in b])
            ]
        )

        obj = self.make_for_example_with_new_config(_NewConf())
        provider = ConfigProvider(obj, configs_objects_map={"ml": obj._ML.conf, "web": obj._WEB.conf})
        config_dict = provider.get_config_dict()

        ml_keys = EXCLUDE_KEYS['web'] - EXCLUDE_KEYS['ml']
        web_app_keys = EXCLUDE_KEYS['ml'] - EXCLUDE_KEYS['web']

        assert config_dict == {
            'ml': {k: f'default_{k}' for k in ml_keys},
            'web': {k: f'default_{k}' for k in web_app_keys}
        }

    def test_get_config_dict_with_condition_exclude_keys(self):
        _fields = []
        _skip_fields = set()
        random_field = None
        random_skip = None
        for cond in EXCLUDE_CONDITION_KEYS:
            for k, v in cond['condition'].items():
                _fields.append((k, type(v), v))
                if not random_field:
                    random_field = k
            if not random_skip:
                random_skip = cond['skip']
            for skip in cond['skip']:
                _fields.append((skip, str, f'default_{skip}'))
                _skip_fields.add(skip)
        _fields.append(('TEST', str, 'default_TEST'))

        _NewConf = make_dataclass('_NewConf', _fields)

        obj = self.make_for_example_with_new_config(_NewConf())
        provider = ConfigProvider(obj, configs_objects_map={"ml": obj._ML.conf, "web": obj._WEB.conf})
        config_dict = provider.get_config_dict()
        assert config_dict == {
            'ml': {k: v for k, t, v in _fields if k not in _skip_fields},
            'web': {k: v for k, t, v in _fields if k not in _skip_fields}
        }

        # Check change condition on valid value
        obj = self.make_for_example_with_new_config(_NewConf(**{random_field: 'NewValue'}))
        provider = ConfigProvider(obj, configs_objects_map={"ml": obj._ML.conf, "web": obj._WEB.conf})
        config_dict = provider.get_config_dict()
        _skip_fields = _skip_fields - set(random_skip)
        assert config_dict == {
            'ml': {k: v if k != random_field else 'NewValue' for k, t, v in _fields if k not in _skip_fields},
            'web': {k: v if k != random_field else 'NewValue' for k, t, v in _fields if k not in _skip_fields}
        }

    def test_set_config_from_dict(self):
        obj = ForExample()
        provider = ConfigProvider(obj, configs_objects_map={"ml": obj._ML.conf, "web": obj._WEB.conf})
        config_dict = {
            'ml': {
                'test': 'test',
                'two_test': 123,
            },
            'web': {
                'three_test': 431,
                'bool_test': True,
            }
        }
        provider.set_config_from_dict(config_dict)
        for k, v in config_dict['ml'].items():
            assert getattr(obj.ml.conf, k) == v
        for k, v in config_dict['web'].items():
            assert getattr(obj.web.conf, k) == v

    def test_set_config_from_dict_with_exclude(self):
        obj = ForExample()
        provider = ConfigProvider(obj, configs_objects_map={"ml": obj._ML.conf, "web": obj._WEB.conf})
        config_dict = {
            'ml': {
                'test': 'test',
                'two_test': 123,
                **{k: f'default_{k}' for k in EXCLUDE_KEYS['ml']}
            },
            'web': {
                'three_test': 431,
                'bool_test': True,
                **{k: f'default_{k}' for k in EXCLUDE_KEYS['web']}
            }
        }
        provider.set_config_from_dict(config_dict)
        for k, v in config_dict['ml'].items():
            if k not in EXCLUDE_KEYS['ml']:
                assert getattr(obj.ml.conf, k) == v
            else:
                assert hasattr(obj.ml.conf, k) is False
        for k, v in config_dict['web'].items():
            if k not in EXCLUDE_KEYS['web']:
                assert getattr(obj.web.conf, k) == v
            else:
                assert hasattr(obj.web.conf, k) is False

    @pytest.mark.parametrize(
        'file_name, load_func',
        [
            ('test_save_to_json.json', json.load),
            ('test_save_to_yaml.yaml', yaml.safe_load),
        ],
        ids=['to_json', 'to_yaml']
    )
    def test_save_to(self, tmp_path_factory, file_name, load_func):
        @dataclass
        class _NewConf(_Conf):
            without_default_int: int
            default_string: str = 'default_string'

        obj = self.make_for_example_with_new_config(_NewConf(123))
        provider = ConfigProvider(obj, configs_objects_map={"ml": obj._ML.conf, "web": obj._WEB.conf})
        provider.save_to_json(tmp_path_factory.getbasetemp() / file_name)

        with open(tmp_path_factory.getbasetemp() / file_name, 'rb') as f:
            data = load_func(f)

        assert data == {
            'ml': {'without_default_int': 123, 'default_string': 'default_string'},
            'web': {'without_default_int': 123, 'default_string': 'default_string'},
            'version': CONFIGS_VERSION,
        }

    @pytest.mark.parametrize(
        'file_name, load_func',
        [
            ('test_save_to_json_with_exclude_keys.json', json.load),
            ('test_save_to_yaml_with_exclude_keys.yaml', yaml.safe_load),
        ],
        ids=['to_json', 'to_yaml']
    )
    def test_save_to_with_exclude_keys(self, tmp_path_factory, file_name, load_func):
        _NewConf = make_dataclass(
            '_NewConf',
            [
                (attr_name, str, field(default=f'default_{attr_name}'))
                for attr_name in set([a for b in EXCLUDE_KEYS.values() for a in b])
            ]
        )

        obj = self.make_for_example_with_new_config(_NewConf())
        provider = ConfigProvider(obj, configs_objects_map={"ml": obj._ML.conf, "web": obj._WEB.conf})
        provider.save_to_json(tmp_path_factory.getbasetemp() / file_name)

        with open(tmp_path_factory.getbasetemp() / file_name, 'rb') as f:
            data = load_func(f)

        ml_keys = EXCLUDE_KEYS['web'] - EXCLUDE_KEYS['ml']
        web_app_keys = EXCLUDE_KEYS['ml'] - EXCLUDE_KEYS['web']

        assert data == {
            'ml': {k: f'default_{k}' for k in ml_keys},
            'web': {k: f'default_{k}' for k in web_app_keys},
            'version': CONFIGS_VERSION,
        }

    @pytest.mark.parametrize(
        'file_name, save_func_name, load_func_name',
        [
            ('test_load_from_json.json', 'save_to_json', 'load_from_json'),
            ('test_load_from_yaml.yaml', 'save_to_yaml', 'load_from_yaml'),
        ],
        ids=['from_json', 'from_yaml']
    )
    def test_load_from(self, tmp_path_factory, file_name, save_func_name, load_func_name):
        @dataclass
        class _NewConf(_Conf):
            without_default_int: int
            default_string: str = 'default_string'

        obj = self.make_for_example_with_new_config(_NewConf(123))
        provider = ConfigProvider(obj, configs_objects_map={"ml": obj._ML.conf, "web": obj._WEB.conf})
        getattr(provider, save_func_name)(tmp_path_factory.getbasetemp() / file_name)

        new_obj = self.make_for_example_with_new_config(_NewConf(321, 'new_string'))
        new_provider = ConfigProvider(
            new_obj,
            configs_objects_map={"ml": new_obj._ML.conf, "web": new_obj._WEB.conf}
        )
        assert new_provider.get_config_dict() == {
            'ml': {'without_default_int': 321, 'default_string': 'new_string'},
            'web': {'without_default_int': 321, 'default_string': 'new_string'}
        }

        getattr(new_provider, load_func_name)(tmp_path_factory.getbasetemp() / file_name)
        assert new_provider.get_config_dict() == {
            'ml': {'without_default_int': 123, 'default_string': 'default_string'},
            'web': {'without_default_int': 123, 'default_string': 'default_string'}
        }

tests/unit_tests/test_configs.py
nxexox/pymlup
import asyncio
import logging
import time
from threading import Thread

import httpx
import pytest
import numpy

from mlup.config import ConfigProvider
from mlup.constants import ModelDataTransformerType
from mlup.ml.model import MLupModel
from mlup.up import UP, Config
from mlup.utils.loop import run_async
from mlup.web.app import MLupWebApp

logger = logging.getLogger('mlup.test')


class TestMLupPublicMethods:
    @staticmethod
    def assert_attributes(conf, obj):
        for key, value in conf.items():
            if isinstance(obj, dict):
                obj_val = obj[key]
            else:
                obj_val = getattr(obj, key)
            assert value == obj_val

    def test_create_mlup(self, print_model):
        up = UP(ml_model=print_model)

        assert isinstance(up.ml, MLupModel)
        assert isinstance(up.web, MLupWebApp)
        assert isinstance(up.config_provider, ConfigProvider)
        assert isinstance(up.conf, Config)

    def test_load_from_dict(self, test_dict_config):
        up = UP.load_from_dict(test_dict_config, load_model=False)

        self.assert_attributes(test_dict_config['ml'], up.ml.conf)
        self.assert_attributes(test_dict_config['web'], up.web.conf)

    def test_to_dict(self, test_dict_config):
        up = UP.load_from_dict(test_dict_config, load_model=False)
        dict_from_up = up.to_dict()

        self.assert_attributes(test_dict_config['ml'], dict_from_up['ml'])
        self.assert_attributes(test_dict_config['web'], dict_from_up['web'])

    def test_load_from_yaml(self, test_yaml_config, test_dict_config):
        up = UP.load_from_yaml(test_yaml_config, load_model=False)

        self.assert_attributes(test_dict_config['ml'], up.ml.conf)
        self.assert_attributes(test_dict_config['web'], up.web.conf)

    def test_to_yaml(self, test_yaml_config, test_dict_config):
        up = UP.load_from_yaml(test_yaml_config, load_model=False)
        up.to_yaml(str(test_yaml_config) + 'test')

        up = UP.load_from_yaml(str(test_yaml_config) + 'test', load_model=False)

        self.assert_attributes(test_dict_config['ml'], up.ml.conf)
        self.assert_attributes(test_dict_config['web'], up.web.conf)

    def test_load_from_json(self, test_json_config, test_dict_config):
        up = UP.load_from_yaml(test_json_config, load_model=False)

        self.assert_attributes(test_dict_config['ml'], up.ml.conf)
        self.assert_attributes(test_dict_config['web'], up.web.conf)

    def test_to_json(self, test_json_config, test_dict_config):
        up = UP.load_from_json(test_json_config, load_model=False)
        up.to_json(str(test_json_config) + 'test')

        up = UP.load_from_json(str(test_json_config) + 'test', load_model=False)

        self.assert_attributes(test_dict_config['ml'], up.ml.conf)
        self.assert_attributes(test_dict_config['web'], up.web.conf)

    @pytest.mark.asyncio
    async def test_predict(self, print_model):
        up = UP(
            ml_model=print_model,
            conf=Config(
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR
            )
        )
        up.ml.load()
        pred = up.predict(X=[[1, 2, 3]])
        assert pred == [[1, 2, 3]]

    @pytest.mark.asyncio
    async def test_async_predict(self, print_model):
        up = UP(
            ml_model=print_model,
            conf=Config(
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR
            )
        )
        up.ml.load()
        pred = await up.async_predict(X=[[1, 2, 3]])
        assert pred == [[1, 2, 3]]

    @pytest.mark.asyncio
    async def test_predict_from_numpy(self, print_model):
        up = UP(
            ml_model=print_model,
            conf=Config(
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
                data_transformer_for_predicted=ModelDataTransformerType.SRC_TYPES,
            )
        )
        up.ml.load()

        predicted_data = up.predict_from(X=[1, 2, 3, 4, 5, 6, 7, 8, 9])
        numpy.array_equal(predicted_data, numpy.array([1, 2, 3, 4, 5, 6, 7, 8, 9]))

    @pytest.mark.asyncio
    async def test_run_web_app_with_daemon_is_False(self, print_model):
        up = UP(
            ml_model=print_model,
            conf=Config(
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR
            )
        )
        up.ml.load()
        up.web.load()

        up.conf.uvicorn_kwargs['loop'] = 'none'

        web_app_thread = Thread(
            target=up.run_web_app,
            kwargs={'daemon': False},
            daemon=False,
            name='MLupWebAppDaemonTestsThread'
        )
        web_app_thread.start()
        await asyncio.sleep(0.5)

        try:
            async with httpx.AsyncClient() as client:
                response = await client.get('http://0.0.0.0:8009/health')
                assert response.status_code == 200
                assert response.json() == {'status': 200}
        finally:
            # Shutdown uvicorn not in main thread
            # https://stackoverflow.com/questions/58010119/are-there-any-better-ways-to-run-uvicorn-in-thread
            up.web._uvicorn_server.should_exit = True
            run_async(
                asyncio.wait_for,
                up.web._uvicorn_server.shutdown(),
                up.web.conf.timeout_for_shutdown_daemon
            )
            web_app_thread.join(timeout=3)

    @pytest.mark.asyncio
    async def test_run_with_daemon_is_True(self, print_model):
        up = UP(
            ml_model=print_model,
            conf=Config(
                port=8011,
                data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            )
        )
        up.ml.load()
        up.web.load()

        up.run_web_app(daemon=True)
        time_run = time.monotonic()
        while not up.web._uvicorn_server and time.monotonic() - time_run < 30.0:
            await asyncio.sleep(0.1)
        while up.web._uvicorn_server.started is False and time.monotonic() - time_run < 30.0:
            await asyncio.sleep(0.1)

        try:
            async with httpx.AsyncClient() as client:
                response = await client.get('http://0.0.0.0:8011/health')
                assert response.status_code == 200
                assert response.json() == {'status': 200}
        finally:
            up.stop_web_app(shutdown_timeout=3)

tests/unit_tests/test_up.py
nxexox/pymlup
from mlup.utils.crypto import generate_unique_id


def test_generate_unique_id():
    count_checks = 10
    assert len({generate_unique_id() for _ in range(count_checks)}) == count_checks

tests/unit_tests/utils/test_crypto.py
nxexox/pymlup
from typing import List
import sys

import pytest

from mlup.constants import IS_X, LoadedFile
from mlup.utils.interspection import analyze_method_params, auto_search_binarization_type


def pred_func_with_X_List(wt, x: List, b: bool = False): pass
def pred_func_with_X_List_of_str(wt, x: List[str], b: bool = False): pass
def pred_func_without_x(wt, y: List, b: bool = False): pass
def pred_func_with_list(wt, x: list, b: bool = False): pass


if sys.version_info.minor >= 9:
    def pred_func_with_list_of_int(wt, x: list[int], b: bool = False): pass
else:
    def pred_func_with_list_of_int(wt, x: List[int], b: bool = False): pass


pred_func_args_without_auto_detect_predict_params = [
    {'name': 'wt', 'required': True},
    {'name': 'x', 'required': True},
    {'name': 'b', 'required': False, 'default': False, 'type': 'bool'},
]
pred_func_args_with_auto_detect_predict_params = [
    {'name': 'wt', 'required': True},
    {'name': 'x', 'required': True, 'type': 'List', IS_X: True},
    {'name': 'b', 'required': False, 'default': False, 'type': 'bool'},
]
pred_func_args_with_auto_detect_predict_params_without_x = [
    {'name': 'wt', 'required': True, 'type': 'List', IS_X: True},
    {'name': 'y', 'required': True},
    {'name': 'b', 'required': False, 'default': False, 'type': 'bool'},
]


class TestAnalyzeMethodParams:

    class ModelClass:
        def pred_func_with_self_and_X_List(self, wt, x: List, b: bool = False): pass
        @classmethod
        def pred_func_with_cls_and_X_List_of_str(cls, wt, x: List[str], b: bool = False): pass
        @staticmethod
        def pred_func_staticmethod_with_list(wt, x: list, b: bool = False): pass
        def pred_func_without_x(self, wt, y: List, b: bool = False): pass

    @pytest.mark.parametrize(
        'pred_func, expected_result',
        [
            pytest.param(
                pred_func_with_X_List, pred_func_args_without_auto_detect_predict_params,
                id='func_with_X_type_List'
            ),
            pytest.param(
                pred_func_with_X_List_of_str, pred_func_args_without_auto_detect_predict_params,
                id='func_with_X_type_List[str]'
            ),
            pytest.param(
                pred_func_with_list, pred_func_args_without_auto_detect_predict_params,
                id='func_with_X_type_list'
            ),
            pytest.param(
                pred_func_with_list_of_int, pred_func_args_without_auto_detect_predict_params,
                id='func_with_X_type_list[int]'
            ),
            pytest.param(
                ModelClass().pred_func_with_self_and_X_List, pred_func_args_without_auto_detect_predict_params,
                id='method_from_obj_with_X_type_List'
            ),
            pytest.param(
                ModelClass.pred_func_with_self_and_X_List, pred_func_args_without_auto_detect_predict_params,
                id='method_fro_cls_with_X_type_List'
            ),
            pytest.param(
                ModelClass.pred_func_with_cls_and_X_List_of_str, pred_func_args_without_auto_detect_predict_params,
                id='classmethod_from_cls_with_cls_and_X_List[str]'
            ),
            pytest.param(
                ModelClass().pred_func_with_cls_and_X_List_of_str, pred_func_args_without_auto_detect_predict_params,
                id='classmethod_from_obj_with_cls_and_X_List[str]'
            ),
            pytest.param(
                ModelClass.pred_func_staticmethod_with_list, pred_func_args_without_auto_detect_predict_params,
                id='staticmethod_from_cls_with_cls_and_X_list'
            ),
            pytest.param(
                ModelClass().pred_func_staticmethod_with_list, pred_func_args_without_auto_detect_predict_params,
                id='staticmethod_from_obj_with_cls_and_X_list'
            ),
        ],
    )
    def test_without_auto_detect_predict_params(self, pred_func, expected_result):
        inspection_params = analyze_method_params(pred_func, auto_detect_predict_params=False)
        assert inspection_params == expected_result

    @pytest.mark.parametrize(
        'pred_func, expected_result',
        [
            pytest.param(
                pred_func_with_X_List, pred_func_args_with_auto_detect_predict_params,
                id='func_with_X_type_List'
            ),
            pytest.param(
                pred_func_with_X_List_of_str, pred_func_args_with_auto_detect_predict_params,
                id='func_with_X_type_List[str]'
            ),
            pytest.param(
                pred_func_with_list, pred_func_args_with_auto_detect_predict_params,
                id='func_with_X_type_list'
            ),
            pytest.param(
                pred_func_with_list_of_int, pred_func_args_with_auto_detect_predict_params,
                id='func_with_X_type_list[int]'
            ),
            pytest.param(
                ModelClass().pred_func_with_self_and_X_List, pred_func_args_with_auto_detect_predict_params,
                id='method_from_obj_with_X_type_List'
            ),
            pytest.param(
                ModelClass.pred_func_with_self_and_X_List, pred_func_args_with_auto_detect_predict_params,
                id='method_from_cls_with_X_type_List'
            ),
            pytest.param(
                ModelClass.pred_func_with_cls_and_X_List_of_str, pred_func_args_with_auto_detect_predict_params,
                id='classmethod_from_cls_with_X_type_List[str]'
            ),
            pytest.param(
                ModelClass().pred_func_with_cls_and_X_List_of_str, pred_func_args_with_auto_detect_predict_params,
                id='classmethod_from_obj_with_X_type_List[str]'
            ),
            pytest.param(
                ModelClass.pred_func_staticmethod_with_list, pred_func_args_with_auto_detect_predict_params,
                id='staticmethod_from_cls_with_X_type_list'
            ),
            pytest.param(
                ModelClass().pred_func_staticmethod_with_list, pred_func_args_with_auto_detect_predict_params,
                id='staticmethod_from_obj_with_X_type_list'
            ),
            pytest.param(
                pred_func_without_x, pred_func_args_with_auto_detect_predict_params_without_x,
                id='func_without_X'
            ),
            pytest.param(
                ModelClass.pred_func_without_x, pred_func_args_with_auto_detect_predict_params_without_x,
                id='method_from_cls_without_X'
            ),
            pytest.param(
                ModelClass().pred_func_without_x, pred_func_args_with_auto_detect_predict_params_without_x,
                id='method_from_obj_without_X'
            ),
        ],
    )
    def test_with_auto_detect_predict_params(self, pred_func, expected_result):
        inspection_params = analyze_method_params(pred_func, auto_detect_predict_params=True)
        assert inspection_params == expected_result

    @pytest.mark.parametrize(
        'pred_func, expected_result',
        [
            pytest.param(
                ModelClass.pred_func_with_self_and_X_List,
                [{'name': 'self', 'required': True}] + pred_func_args_without_auto_detect_predict_params,
                id='method_with_X_List'
            ),
            pytest.param(
                ModelClass.pred_func_with_cls_and_X_List_of_str, pred_func_args_without_auto_detect_predict_params,
                id='classmethod_from_cls_with_X_List[str]'
            ),
            pytest.param(
                ModelClass.pred_func_staticmethod_with_list, pred_func_args_without_auto_detect_predict_params,
                id='staticmethod_from_cls_with_X_list'
            ),
            pytest.param(
                ModelClass.pred_func_without_x,
                [
                    {'name': 'self', 'required': True},
                    {'name': 'wt', 'required': True},
                    {'name': 'y', 'required': True},
                    {'name': 'b', 'required': False, 'default': False, 'type': 'bool'},
                ],
                id='method_from_cls_without_X'
            ),
        ],
    )
    def test_without_ignore_self(self, pred_func, expected_result):
        inspection_params = analyze_method_params(pred_func, auto_detect_predict_params=False, ignore_self=False)
        assert inspection_params == expected_result


class TestAutoSearchBinarizationType:
    @pytest.mark.parametrize(
        'loaded_file',
        [
            LoadedFile(),
            LoadedFile(''),
            LoadedFile(path=''),
            LoadedFile('not valid bytes', 'not valid path.test'),
            LoadedFile(path='joblib.joblib')
        ],
        ids=['empty', 'empty_raw', 'empty_path', 'not_valid_data', 'joblib']
    )
    def test_not_found_binarizer(self, loaded_file):
        assert auto_search_binarization_type(loaded_file) is None

    @pytest.mark.parametrize(
        'loaded_file',
        [
            LoadedFile(path='pickle.pckl'),
            LoadedFile(path='model.onnx'),
            LoadedFile(b'PK\x03', path='model.h5'),
            LoadedFile(b'PK\x03', path='model.pth'),
            LoadedFile('\n\n\n\nversion=\n', path='.txt')
        ],
        ids=['pickle', 'onnx', 'tensorflow', 'torch', 'lightgbm'],
    )
    def test_found_binarizer(self, loaded_file):
        assert auto_search_binarization_type(loaded_file) is not None

tests/unit_tests/utils/test_interspection.py
nxexox/pymlup
import copy
from typing import Optional
from unittest import TestCase

from pydantic import BaseModel as PydanticBaseModel

from mlup.constants import IS_X, DEFAULT_X_ARG_NAME, WebAppArchitecture
from mlup.up import UP, Config
from mlup.web.api_docs import make_columns_object_openapi_scheme, generate_openapi_schema


assertDictEqual = TestCase().assertDictEqual


src_columns = [
    {"name": "Float", "type": "float"},
    {"name": "FloatDefault", "type": "float", "default": 1.4},
    {"name": "FloatRequired", "type": "float", "required": True},
    {"name": "FloatNotRequired", "type": "float", "required": False},
    {"name": "FloatNotRequiredDefault", "type": "float", "required": False, "default": 1.4},
    {"name": "FloatRequiredDefault", "type": "float", "required": True, "default": 1.4},

    {"name": "Int", "type": "int"},
    {"name": "IntDefault", "type": "int", "default": 4},
    {"name": "IntRequired", "type": "int", "required": True},
    {"name": "IntNotRequired", "type": "int", "required": False},
    {"name": "IntNotRequiredDefault", "type": "int", "required": False, "default": 4},
    {"name": "IntRequiredDefault", "type": "int", "required": True, "default": 4},

    {"name": "Str", "type": "str"},
    {"name": "StrDefault", "type": "str", "default": "str"},
    {"name": "StrRequired", "type": "str", "required": True},
    {"name": "StrNotRequired", "type": "str", "required": False},
    {"name": "StrNotRequiredDefault", "type": "str", "required": False, "default": "str"},
    {"name": "StrRequiredDefault", "type": "str", "required": True, "default": "str"},

    {"name": "Bool", "type": "bool"},
    {"name": "BoolDefault", "type": "bool", "default": True},
    {"name": "BoolRequired", "type": "bool", "required": True},
    {"name": "BoolNotRequired", "type": "bool", "required": False},
    {"name": "BoolNotRequiredDefault", "type": "bool", "required": False, "default": True},
    {"name": "BoolRequiredDefault", "type": "bool", "required": True, "default": True},
]


openapi_cols = {
    'Float': {'title': 'Float', 'type': 'number'},
    'FloatDefault': {'title': 'Default Float', 'type': 'number', 'default': 1.4},
    'FloatRequired': {'title': 'Float', 'type': 'number'},
    'FloatNotRequired': {'title': 'Optional Float', 'type': 'number'},
    'FloatNotRequiredDefault': {'title': 'Default Optional Float', 'type': 'number', 'default': 1.4},
    'FloatRequiredDefault': {'title': 'Default Float', 'type': 'number', 'default': 1.4},
    'Int': {'title': 'Int', 'type': 'integer'},
    'IntDefault': {'title': 'Default Int', 'type': 'integer', 'default': 4},
    'IntRequired': {'title': 'Int', 'type': 'integer'},
    'IntNotRequired': {'title': 'Optional Int', 'type': 'integer'},
    'IntNotRequiredDefault': {'title': 'Default Optional Int', 'type': 'integer', 'default': 4},
    'IntRequiredDefault': {'title': 'Default Int', 'type': 'integer', 'default': 4},
    'Str': {'title': 'Str', 'type': 'string'},
    'StrDefault': {'title': 'Default Str', 'type': 'string', 'default': 'str'},
    'StrRequired': {'title': 'Str', 'type': 'string'},
    'StrNotRequired': {'title': 'Optional Str', 'type': 'string'},
    'StrNotRequiredDefault': {'title': 'Default Optional Str', 'type': 'string', 'default': 'str'},
    'StrRequiredDefault': {'title': 'Default Str', 'type': 'string', 'default': 'str'},
    'Bool': {'title': 'Bool', 'type': 'boolean'},
    'BoolDefault': {'title': 'Default Bool', 'type': 'boolean', 'default': True},
    'BoolRequired': {'title': 'Bool', 'type': 'boolean'},
    'BoolNotRequired': {'title': 'Optional Bool', 'type': 'boolean'},
    'BoolNotRequiredDefault': {'title': 'Default Optional Bool', 'type': 'boolean', 'default': True},
    'BoolRequiredDefault': {'title': 'Default Bool', 'type': 'boolean', 'default': True}
}
openapi_required_cols = [
    'Float', 'FloatDefault', 'FloatRequired', 'FloatRequiredDefault',
    'Int', 'IntDefault', 'IntRequired', 'IntRequiredDefault',
    'Str', 'StrDefault', 'StrRequired', 'StrRequiredDefault',
    'Bool', 'BoolDefault', 'BoolRequired', 'BoolRequiredDefault'
]

openapi_full_scheme = {
    'openapi': '3.0.2',
    'info': {
        'title': 'MLup web application with model: MyFirstMLupModel v1.0.0.0.',
        'description': 'Web application for use MyFirstMLupModel v1.0.0.0 in web.',
        'version': '1.0.0.0'
    },
    'paths': {
        '/health': {
            'get': {
                'summary': 'Http Health',
                'operationId': 'http_health_health_get',
                'responses': {
                    '200': {
                        'description': 'Successful Response',
                        'content': {
                            'application/json': {
                                'schema': {}
                            }
                        }
                    }
                }
            },
            'options': {
                'summary': 'Http Health',
                'operationId': 'http_health_health_options',
                'responses': {
                    '200': {
                        'description': 'Successful Response',
                        'content': {
                            'application/json': {
                                'schema': {}
                            }
                        }
                    }
                }
            },
            'head': {
                'summary': 'Http Health',
                'operationId': 'http_health_health_head',
                'responses': {
                    '200': {
                        'description': 'Successful Response',
                        'content': {
                            'application/json': {
                                'schema': {}
                            }
                        }
                    },
                }
            },
        },
        '/info': {
            'get': {
                'summary': 'Info',
                'operationId': 'info_info_get',
                'responses': {
                    '200': {
                        'description': 'Successful Response',
                        'content': {
                            'application/json': {
                                'schema': {}
                            }
                        }
                    }
                }
            }
        },
        '/predict': {
            'post': {
                'summary': 'Predict',
                'operationId': 'predict_predict_post',
                'requestBody': {
                    'content': {
                        'application/json': {
                            'schema': {
                                '$ref': '#/components/schemas/PredictItems'
                            }
                        }
                    },
                    'required': True
                },
                'responses': {
                    '200': {
                        'description': 'Successful Response',
                        'content': {
                            'application/json': {
                                'schema': {}
                            }
                        }
                    },
                    '422': {
                        'content': {
                            'application/json': {
                                'schema': {
                                    '$ref': '#/components/schemas/ApiErrorResponse'
                                }
                            }
                        },
                        'description': 'Error with validation input data'
                    },
                    '429': {
                        'content': {
                            'application/json': {
                                'schema': {
                                    '$ref': '#/components/schemas/ApiErrorResponse'
                                }
                            }
                        },
                        'description': 'Throttling input request error'
                    },
                    '500': {
                        'content': {
                            'application/json': {
                                'schema': {
                                    '$ref': '#/components/schemas/ApiErrorResponse'
                                }
                            }
                        },
                        'description': 'Error with predict process exception'
                    }
                }
            }
        }
    },
    'components': {
        'schemas': {
            'ApiError': {
                'title': 'ApiError',
                'required': ['loc', 'msg', 'type'],
                'type': 'object',
                'properties': {
                    'loc': {
                        'title': 'Loc',
                        'type': 'array',
                        'items': {}
                    },
                    'msg': {
                        'title': 'Msg',
                        'type': 'string'
                    },
                    'type': {
                        'title': 'Type',
                        'type': 'string'
                    }
                }
            },
            'ApiErrorResponse': {
                'title': 'ApiErrorResponse',
                'required': ['detail'],
                'type': 'object',
                'properties': {
                    'detail': {
                        'title': 'Detail',
                        'type': 'array',
                        'items': {
                            '$ref': '#/components/schemas/ApiError'
                        }
                    },
                    'predict_id': {
                        'title': 'Predict Id',
                        'type': 'string'
                    }
                },
            },
            'PredictItems': {
                'title': 'Predict items',
                'required': ['X'],
                'type': 'object',
                'properties': {
                    'X': {
                        'title': 'List',
                        'type': 'array',
                        'items': {
                            '$ref': '#/components/schemas/ColumnsForPredict'
                        }
                    },
                    'test_param': {
                        'title': 'Default Optional Bool',
                        'type': 'boolean',
                        'default': False
                    }
                }
            },
            'ColumnsForPredict': {
                'title': 'Columns for predict',
                'required': [],
                'type': 'object',
                'properties': {}
            }
        }
    }
}
openapi_is_long_predict_method = {
    '/get-predict/{predict_id}': {
        'get': {
            'summary': 'Get Predict Result',
            'operationId': 'get_predict_result_get_predict__predict_id__get',
            'parameters': [
                {
                    'required': True,
                    'schema': {'title': 'Predict Id', 'type': 'string'},
                    'name': 'predict_id',
                    'in': 'path'
                }
            ],
            'responses': {
                '200': {'description': 'Successful Response', 'content': {'application/json': {'schema': {}}}},
                '422': {
                    'description': 'Error with validation input data',
                    'content': {'application/json': {'schema': {'$ref': '#/components/schemas/ApiErrorResponse'}}}
                },
                '429': {
                    'description': 'Throttling input request error',
                    'content': {'application/json': {'schema': {'$ref': '#/components/schemas/ApiErrorResponse'}}}
                },
                '500': {
                    'description': 'Error with predict process exception',
                    'content': {'application/json': {'schema': {'$ref': '#/components/schemas/ApiErrorResponse'}}}
                }
            }
        }
    }
}


def test_make_columns_object_openapi_scheme():
    pred_cols, pred_required_cols = make_columns_object_openapi_scheme(src_columns)

    assertDictEqual(pred_cols, openapi_cols)
    assert pred_required_cols == openapi_required_cols


def test_make_columns_object_openapi_scheme_with_IS_X():
    test_columns = copy.deepcopy(src_columns)
    test_columns[0][IS_X] = True
    pred_cols, pred_required_cols = make_columns_object_openapi_scheme(test_columns)

    test_openapi_cols = copy.deepcopy(openapi_cols)
    test_openapi_cols['Float']['items'] = {"$ref": "#/components/schemas/ColumnsForPredict"}
    assertDictEqual(pred_cols, test_openapi_cols)
    assert pred_required_cols == openapi_required_cols


def test_generate_openapi_schema(print_model):
    up = UP(ml_model=print_model)
    # Load ml with web
    up.ml.load()
    up.web.load()
    generated_scheme = generate_openapi_schema(up.web.app, up.ml)
    openapi_full_scheme['openapi'] = generated_scheme['openapi']
    assertDictEqual(generated_scheme, openapi_full_scheme)


def test_generate_openapi_scheme_with_default_X_and_custom_columns(print_model):
    up = UP(
        ml_model=print_model,
        conf=Config(
            auto_detect_predict_params=False,
            columns=src_columns,
        )
    )
    # Load ml with web
    up.ml.load()
    up.web.load()
    generated_scheme = generate_openapi_schema(up.web.app, up.ml)

    _openapi_full_scheme = copy.deepcopy(openapi_full_scheme)
    _scheme_predict_items = _openapi_full_scheme['components']['schemas']['PredictItems']
    _scheme_predict_items['required'] = [DEFAULT_X_ARG_NAME]
    _scheme_predict_items['properties'] = {
        DEFAULT_X_ARG_NAME: {
            'title': 'Data for predict',
            'type': 'array',
            'items': {
                '$ref': '#/components/schemas/ColumnsForPredict'
            }
        }
    }
    _scheme_columns_for_predict = _openapi_full_scheme['components']['schemas']['ColumnsForPredict']
    _scheme_columns_for_predict['required'] = openapi_required_cols
    _scheme_columns_for_predict['properties'] = openapi_cols

    _openapi_full_scheme['openapi'] = generated_scheme['openapi']
    assertDictEqual(generated_scheme, _openapi_full_scheme)


def test_generate_openapi_scheme_with_default_X_without_columns(print_model):
    up = UP(
        ml_model=print_model,
        conf=Config(
            auto_detect_predict_params=False,
        )
    )
    # Load ml with web
    up.ml.load()
    up.web.load()
    generated_scheme = generate_openapi_schema(up.web.app, up.ml)

    _openapi_full_scheme = copy.deepcopy(openapi_full_scheme)
    _scheme_predict_items = _openapi_full_scheme['components']['schemas']['PredictItems']
    _scheme_predict_items['required'] = [DEFAULT_X_ARG_NAME]
    _scheme_predict_items['properties'] = {
        DEFAULT_X_ARG_NAME: {
            'title': 'Data for predict',
            'type': 'array',
            'items': {
                '$ref': '#/components/schemas/ColumnsForPredict'
            }
        }
    }

    openapi_full_scheme['openapi'] = generated_scheme['openapi']
    assertDictEqual(generated_scheme, _openapi_full_scheme)


def test_generate_openapi_scheme_with_auto_analyze_X_with_custom_columns(print_model):
    up = UP(
        ml_model=print_model,
        conf=Config(
            auto_detect_predict_params=True,
            columns=src_columns,
        )
    )
    # Load ml with web
    up.ml.load()
    up.web.load()
    generated_scheme = generate_openapi_schema(up.web.app, up.ml)

    _openapi_full_scheme = copy.deepcopy(openapi_full_scheme)
    _scheme_predict_items = _openapi_full_scheme['components']['schemas']['PredictItems']
    _scheme_predict_items['required'] = ['X']
    _scheme_predict_items['properties'] = {
        'X': {'title': 'List', 'type': 'array', 'items': {'$ref': '#/components/schemas/ColumnsForPredict'}},
        'test_param': {'title': 'Default Optional Bool', 'type': 'boolean', 'default': False},
    }
    _scheme_columns_for_predict = _openapi_full_scheme['components']['schemas']['ColumnsForPredict']
    _scheme_columns_for_predict['required'] = openapi_required_cols
    _scheme_columns_for_predict['properties'] = openapi_cols

    _openapi_full_scheme['openapi'] = generated_scheme['openapi']
    assertDictEqual(generated_scheme, _openapi_full_scheme)


def test_generate_openapi_scheme_with_auto_analyze_X_without_columns(print_model):
    up = UP(
        ml_model=print_model,
        conf=Config(auto_detect_predict_params=True)
    )
    # Load ml with web
    up.ml.load()
    up.web.load()
    generated_scheme = generate_openapi_schema(up.web.app, up.ml)

    openapi_full_scheme['openapi'] = generated_scheme['openapi']
    assertDictEqual(generated_scheme, openapi_full_scheme)


def test_generate_openapi_scheme_with_is_long_predict(print_model):
    up = UP(
        ml_model=print_model,
        conf=Config(auto_detect_predict_params=False)
    )
    # Load ml with web
    up.ml.load()
    up.web.conf.mode = WebAppArchitecture.worker_and_queue
    up.web.conf.is_long_predict = True
    up.web.load()
    generated_scheme = generate_openapi_schema(up.web.app, up.ml)

    _openapi_full_scheme = copy.deepcopy(openapi_full_scheme)
    _openapi_full_scheme['paths'].update(openapi_is_long_predict_method)
    _scheme_predict_items = _openapi_full_scheme['components']['schemas']['PredictItems']
    _scheme_predict_items['required'] = [DEFAULT_X_ARG_NAME]
    _scheme_predict_items['properties'] = {
        DEFAULT_X_ARG_NAME: {
            'title': 'Data for predict',
            'type': 'array',
            'items': {
                '$ref': '#/components/schemas/ColumnsForPredict'
            }
        }
    }

    _openapi_full_scheme['openapi'] = generated_scheme['openapi']
    assertDictEqual(generated_scheme, _openapi_full_scheme)


def test_generate_openapi_scheme_with_add_custom_handler(print_model):
    class CustomRequestData(PydanticBaseModel):
        i: int
        f: Optional[float] = None
        s: str = 'test string'
        b: Optional[bool] = False

    async def test_api_handler(item_id: int, data: CustomRequestData):
        return data.dict()

    up = UP(
        ml_model=print_model,
        conf=Config(auto_detect_predict_params=False)
    )
    # Load ml with web
    up.ml.load()
    # Add custom handler
    up.web.load()
    up.web.app.add_api_route("/test-api-handler/{item_id}", test_api_handler, methods=["POST"])
    generated_scheme = generate_openapi_schema(up.web.app, up.ml)

    _openapi_full_scheme = copy.deepcopy(openapi_full_scheme)
    _scheme_predict_items = _openapi_full_scheme['components']['schemas']['PredictItems']
    _scheme_predict_items['required'] = [DEFAULT_X_ARG_NAME]
    _scheme_predict_items['properties'] = {
        DEFAULT_X_ARG_NAME: {
            'title': 'Data for predict',
            'type': 'array',
            'items': {
                '$ref': '#/components/schemas/ColumnsForPredict'
            }
        }
    }
    _openapi_full_scheme['paths']['/test-api-handler/{item_id}'] = {
        'post': {
            'summary': 'Test Api Handler',
            'operationId': 'test_api_handler_test_api_handler__item_id__post',
            'parameters': [
                {
                    'required': True,
                    'schema': {'title': 'Item Id', 'type': 'integer'},
                    'name': 'item_id',
                    'in': 'path'
                }
            ],
            'requestBody': {
                'content': {'application/json': {'schema': {'$ref': '#/components/schemas/CustomRequestData'}}},
                'required': True
            },
            'responses': {
                '200': {'description': 'Successful Response', 'content': {'application/json': {'schema': {}}}},
                '422': {
                    'description': 'Error with validation input data',
                    'content': {'application/json': {'schema': {'$ref': '#/components/schemas/ApiErrorResponse'}}}
                },
                '429': {
                    'description': 'Throttling input request error',
                    'content': {'application/json': {'schema': {'$ref': '#/components/schemas/ApiErrorResponse'}}}
                },
                '500': {
                    'description': 'Error with predict process exception',
                    'content': {'application/json': {'schema': {'$ref': '#/components/schemas/ApiErrorResponse'}}}
                }
            }
        }
    }
    _openapi_full_scheme['components']['schemas']['CustomRequestData'] = {
        'title': 'CustomRequestData',
        'required': ['i'],
        'type': 'object',
        'properties': {
            'i': {'title': 'I', 'type': 'integer'},
            'f': {'title': 'F', 'type': 'number'},
            's': {'title': 'S', 'type': 'string', 'default': 'test string'},
            'b': {'title': 'B', 'type': 'boolean', 'default': False}
        }
    }

    _openapi_full_scheme['openapi'] = generated_scheme['openapi']
    assertDictEqual(generated_scheme, _openapi_full_scheme)

tests/unit_tests/web/test_api_docs.py
nxexox/pymlup
import copy
from typing import List, Any

import pytest
from pydantic import BaseModel as PydanticBaseModel, create_model, ValidationError

from mlup.constants import IS_X, DEFAULT_X_ARG_NAME
from mlup.ml.model import MLupModel, ModelConfig
from mlup.web.api_validators import (
    make_map_pydantic_columns,
    create_pydantic_predict_model,
)


column_types_map = {
    'int': int,
    'float': float,
    'bool': bool,
    'str': str,
    'list': list,
}
src_columns = [
    {"name": "Float", "type": "float"},
    {"name": "FloatDefault", "type": "float", "default": 1.4},
    {"name": "FloatRequired", "type": "float", "required": True},
    {"name": "FloatNotRequired", "type": "float", "required": False},
    {"name": "FloatNotRequiredDefault", "type": "float", "required": False, "default": 1.4},
    {"name": "FloatRequiredDefault", "type": "float", "required": True, "default": 1.4},

    {"name": "Int", "type": "int"},
    {"name": "IntDefault", "type": "int", "default": 4},
    {"name": "IntRequired", "type": "int", "required": True},
    {"name": "IntNotRequired", "type": "int", "required": False},
    {"name": "IntNotRequiredDefault", "type": "int", "required": False, "default": 4},
    {"name": "IntRequiredDefault", "type": "int", "required": True, "default": 4},

    {"name": "Str", "type": "str"},
    {"name": "StrDefault", "type": "str", "default": "str"},
    {"name": "StrRequired", "type": "str", "required": True},
    {"name": "StrNotRequired", "type": "str", "required": False},
    {"name": "StrNotRequiredDefault", "type": "str", "required": False, "default": "str"},
    {"name": "StrRequiredDefault", "type": "str", "required": True, "default": "str"},

    {"name": "Bool", "type": "bool"},
    {"name": "BoolDefault", "type": "bool", "default": True},
    {"name": "BoolRequired", "type": "bool", "required": True},
    {"name": "BoolNotRequired", "type": "bool", "required": False},
    {"name": "BoolNotRequiredDefault", "type": "bool", "required": False, "default": True},
    {"name": "BoolRequiredDefault", "type": "bool", "required": True, "default": True},
]


def test_make_map_pydantic_columns():
    cols_configs, validators = make_map_pydantic_columns(src_columns)

    for col_config in src_columns:
        pred_col_type, pred_field_info = cols_configs.pop(col_config["name"])

        assert pred_col_type is column_types_map[col_config["type"]]
        assert pred_field_info.title == col_config["name"]
        if 'default' in col_config:
            assert pred_field_info.default is col_config["default"]
        elif col_config.get('required', True) is False:
            assert pred_field_info.default is None
        else:
            assert pred_field_info.default is Ellipsis

    assert len(cols_configs) == 0
    assert len(validators) == 0


def test_make_map_pydantic_validation():
    cols_configs, validators = make_map_pydantic_columns(src_columns)

    for col_config in src_columns:
        pred_col_type, pred_field_info = cols_configs.pop(col_config["name"])

        _test_pydantic_model = create_model(
            "_TestPydanticModel",
            **{col_config["name"]: (pred_col_type, pred_field_info)},
        )

        # Check valid type
        _test_pydantic_model(**{col_config["name"]: pred_col_type(1)})
        # Check not valid type
        try:
            not_valid_value = list
            _test_pydantic_model(**{col_config["name"]: not_valid_value})
            pytest.fail('Not raise error')
        except ValidationError as e:
            msg_str = str(e.raw_errors[0].exc.msg_template)
            if pred_col_type is str:
                assert msg_str == 'str type expected'
            elif pred_col_type is bool:
                assert msg_str == 'value could not be parsed to a boolean'
            else:
                assert msg_str.startswith(f'value is not a valid {col_config["type"]}')

        # Check required
        if col_config.get("required", True):
            # Check valid value
            _test_pydantic_model(**{col_config["name"]: pred_col_type(1)})
            # Check none value
            try:
                _test_pydantic_model(**{col_config["name"]: None})
                pytest.fail('Not raise error')
            except ValidationError as e:
                assert str(e.raw_errors[0].exc).startswith('none is not an allowed value')
        # Check not required
        else:
            # Check not exists value
            _test_pydantic_model()
            # Check None value
            try:
                _test_pydantic_model(**{col_config["name"]: None})
            except ValidationError as e:
                assert str(e.raw_errors[0].exc.msg_template) == 'none is not an allowed value'

    assert len(cols_configs) == 0
    assert len(validators) == 0


@pytest.mark.parametrize(
    'model_for_columns',
    [None, create_model("_TestPydanticModel")],
    ids=['WITHOUT_COLUMNS_MODEL', 'WITH_COLUMNS_MODEL']
)
def test_make_map_pydantic_columns_with_IS_X(model_for_columns):
    test_columns = copy.deepcopy(src_columns)
    test_columns[0][IS_X] = True

    _kwargs = dict(src_columns=test_columns)
    if model_for_columns:
        _kwargs['x_model'] = model_for_columns

    cols_configs, validators = make_map_pydantic_columns(**_kwargs)

    for col_config in test_columns:
        pred_col_type, pred_field_info = cols_configs.pop(col_config["name"])
        col_is_X = IS_X in col_config

        if col_is_X:
            if model_for_columns:
                assert pred_col_type is List[model_for_columns]
            else:
                assert pred_col_type is List[Any]
        else:
            assert pred_col_type is column_types_map[col_config["type"]]
        assert pred_field_info.title == col_config["name"]
        if 'default' in col_config:
            assert pred_field_info.default is col_config["default"]
        elif col_config.get('required', True) is False:
            assert pred_field_info.default is None
        else:
            assert pred_field_info.default is Ellipsis

    assert len(cols_configs) == 0
    assert len(validators) == 0


@pytest.mark.parametrize(
    'auto_detect_predict_params, x_param_name',
    [(True, 'X'), (False, DEFAULT_X_ARG_NAME)],
    ids=['auto_detect_predict_params=True', 'auto_detect_predict_params=False']
)
@pytest.mark.parametrize(
    'column_validation, columns, data, expected_data',
    [
        (False, src_columns, [1, 2, 3], [1, 2, 3]),
        (
            True,
            [src_columns[0], src_columns[6], src_columns[12]],
            [{'Float': 1.0, 'Int': 1, 'Str': '1', 'NotExistsKey': 10}],
            [{'Float': 1.0, 'Int': 1, 'Str': '1'}]
        ),
    ],
    ids=['column_validation=False', 'column_validation=True']
)
def test_create_pydantic_predict_model_valid(
    model_with_x,
    auto_detect_predict_params: bool,
    x_param_name: str,
    column_validation: bool,
    columns: List,
    data: List,
    expected_data: List,
):
    ml = MLupModel(
        ml_model=model_with_x,
        conf=ModelConfig(
            auto_detect_predict_params=auto_detect_predict_params, columns=columns
        )
    )
    ml.load()
    pred_pydantic_model = create_pydantic_predict_model(ml, column_validation=column_validation)

    # Check valid value
    data_for_pred = {x_param_name: data}
    ddt = pred_pydantic_model(**data_for_pred)
    assert ddt.dict() == {x_param_name: expected_data}

    # Check empty value
    try:
        pred_pydantic_model(not_exists_key=data).dict()
        pytest.fail('Not raise error')
    except ValidationError as e:
        assert str(e) == f'1 validation error for MLupRequestPredictPydanticModel\n{x_param_name}\n  ' \
                         'field required (type=value_error.missing)'

    # Check not valid value
    try:
        pred_pydantic_model(**{x_param_name: 1}).dict()
        pytest.fail('Not raise error')
    except ValidationError as e:
        assert str(e) == f'1 validation error for MLupRequestPredictPydanticModel\n{x_param_name}\n  ' \
                         'value is not a valid list (type=type_error.list)'


@pytest.mark.parametrize(
    'auto_detect_predict_params, x_param_name',
    [(True, 'X'), (False, DEFAULT_X_ARG_NAME)],
    ids=['auto_detect_predict_params=True', 'auto_detect_predict_params=False']
)
@pytest.mark.parametrize(
    'data, expected_data',
    [
        ([{'test_column': 1}], [{'test_column': '1'}]),
        ([{'test_column': 1, 'NotExistsKey': 10}], [{'test_column': '1'}]),
    ],
    ids=['data={"test_column": 1}', 'data={"test_column": 1, "not_exists_key": 1}']
)
def test_create_pydantic_predict_model_custom_column_pydantic_model(
    model_with_x,
    auto_detect_predict_params: bool,
    x_param_name: str,
    data: List,
    expected_data: List,
):
    ml = MLupModel(
        ml_model=model_with_x,
        conf=ModelConfig(
            auto_detect_predict_params=auto_detect_predict_params, columns=src_columns
        )
    )
    ml.load()

    class TestPydanticModel(PydanticBaseModel):
        test_column: str

    pred_pydantic_model = create_pydantic_predict_model(
        ml, column_validation=False, custom_column_pydantic_model=TestPydanticModel
    )

    # Check valid value
    data_for_pred = {x_param_name: data}
    ddt = pred_pydantic_model(**data_for_pred)
    assert ddt.dict() == {x_param_name: expected_data}

    # Check empty value
    try:
        pred_pydantic_model(not_exists_key=data).dict()
        pytest.fail('Not raise error')
    except ValidationError as e:
        assert str(e) == f'1 validation error for MLupRequestPredictPydanticModel\n{x_param_name}\n  ' \
                         'field required (type=value_error.missing)'

    # Check not valid
    try:
        pred_pydantic_model(**{x_param_name: 1}).dict()
        pytest.fail('Not raised error')
    except ValidationError as e:
        assert str(e) == f'1 validation error for MLupRequestPredictPydanticModel\n{x_param_name}\n  ' \
                         'value is not a valid list (type=type_error.list)'

tests/unit_tests/web/test_api_validators.py
nxexox/pymlup
import asyncio
from threading import Thread
from typing import Dict
from unittest import TestCase

import httpx
import pytest
from fastapi import FastAPI

from mlup.constants import ModelDataTransformerType, ITEM_ID_COL_NAME, WebAppArchitecture
from mlup.errors import WebAppLoadError
from mlup.ml.model import MLupModel, ModelConfig
from mlup.utils.loop import run_async, create_async_task
from mlup.web.app import MLupWebApp, WebAppConfig


assertDictEqual = TestCase().assertDictEqual


DEFAULT_API_INFO_METHOD_DATA = {
    'model_info': {
        'name': 'MyFirstMLupModel',
        'version': '1.0.0.0',
        'type': 'sklearn',
        'columns': None
    },
    'web_app_info': {
        'version': '1.0.0.0',
    },
}


DEFAULT_API_DEBUG_INFO_METHOD_DATA = {
    **DEFAULT_API_INFO_METHOD_DATA,
    'web_app_config': {
        'host': 'localhost',
        'port': 8009,
        'web_app_version': '1.0.0.0',
        'column_validation': False,
        'custom_column_pydantic_model': None,
        'mode': 'mlup.web.architecture.directly_to_predict.DirectlyToPredictArchitecture',
        'max_queue_size': 100,
        'ttl_predicted_data': 60,
        'ttl_client_wait': 30.0,
        'min_batch_len': 10,
        'batch_worker_timeout': 1.0,
        'is_long_predict': False,
        'show_docs': True,
        'debug': True,
        'throttling_max_requests': None,
        'throttling_max_request_len': None,
        'timeout_for_shutdown_daemon': 3.0,
        'item_id_col_name': 'mlup_item_id'
    },
    'model_config': {
        'name': 'MyFirstMLupModel',
        'version': '1.0.0.0',
        'type': 'sklearn',
        'columns': None,
        'predict_method_name': 'predict',
        'auto_detect_predict_params': True,
        'storage_type': 'mlup.ml.storage.memory.MemoryStorage',
        'binarization_type': 'auto',
        'use_thread_loop': True,
        'max_thread_loop_workers': None,
        'data_transformer_for_predict': 'mlup.ml.data_transformers.numpy_data_transformer.NumpyDataTransformer',
        'data_transformer_for_predicted': 'mlup.ml.data_transformers.numpy_data_transformer.NumpyDataTransformer',
        'dtype_for_predict': None,
    }
}


def test_create_web_app(print_model):
    mlup_model = MLupModel(ml_model=print_model)
    mlup_web_app = MLupWebApp(ml=mlup_model)

    assert mlup_web_app.conf.host == '0.0.0.0'
    assert mlup_web_app.conf.port == 8009
    assert mlup_web_app.conf.web_app_version == '1.0.0.0'
    assert mlup_web_app.conf.column_validation is False
    assert mlup_web_app.conf.custom_column_pydantic_model is None
    assert mlup_web_app.conf.mode is WebAppArchitecture.directly_to_predict
    assert mlup_web_app.conf.max_queue_size == 100
    assert mlup_web_app.conf.ttl_predicted_data == 60
    assert mlup_web_app.conf.ttl_client_wait == 30.0
    assert mlup_web_app.conf.min_batch_len == 10
    assert mlup_web_app.conf.batch_worker_timeout == 1.0
    assert mlup_web_app.conf.is_long_predict is False
    assert mlup_web_app.conf.throttling_max_requests is None
    assert mlup_web_app.conf.throttling_max_request_len is None
    assert mlup_web_app.conf.timeout_for_shutdown_daemon == 3.0
    assert mlup_web_app.conf.item_id_col_name == ITEM_ID_COL_NAME


def test_web_app_attribute(print_model):
    mlup_model = MLupModel(ml_model=print_model)
    mlup_web_app = MLupWebApp(ml=mlup_model)
    mlup_model.load()

    try:
        mlup_web_app.app
        pytest.fail('Not raised error')
    except WebAppLoadError as e:
        assert str(e) == 'web nor creating. Please call web.load().'

    mlup_web_app.load()
    assert isinstance(mlup_web_app.app, FastAPI)


@pytest.mark.parametrize(
    'params, needed_api_points',
    [
        (
            {'is_long_predict': False, 'mode': WebAppArchitecture.directly_to_predict},
            {'/predict': {'POST', }, '/info': {'GET', }}
        ),
        (
            {'is_long_predict': True, 'mode': WebAppArchitecture.worker_and_queue},
            {'/predict': {'POST', }, '/info': {'GET', }, '/get-predict/{predict_id}': {'GET', }}
        ),
    ],
    ids=['is_long_predict=False', 'is_long_predict=True']
)
def test_load_web_app_check_api_points(print_model, params, needed_api_points):
    mlup_model = MLupModel(ml_model=print_model)
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(**params)
    )
    mlup_model.load()
    mlup_web_app.load()
    assert isinstance(mlup_web_app.app, FastAPI)

    for api_point in mlup_web_app.app.routes:
        if api_point.path in needed_api_points:
            assert needed_api_points[api_point.path] == api_point.methods
            del needed_api_points[api_point.path]

    assert len(needed_api_points) == 0


@pytest.mark.parametrize(
    'web_app_params, ml_params, error',
    [
        ({'throttling_max_requests': -1}, {},
         'The param throttling_max_requests must be greater than 0. Now it is -1.'),
        ({'throttling_max_requests': 0}, {},
         'The param throttling_max_requests must be greater than 0. Now it is 0.'),
        ({'throttling_max_requests': 0.99}, {},
         'The param throttling_max_requests must be greater than 0. Now it is 0.99.'),
        ({'throttling_max_request_len': -1}, {},
         'The param throttling_max_request_len must be greater than 0. Now it is -1.'),
        ({'throttling_max_request_len': 0}, {},
         'The param throttling_max_request_len must be greater than 0. Now it is 0.'),
        ({'throttling_max_request_len': 0.9}, {},
         'The param throttling_max_request_len must be greater than 0. Now it is 0.9.'),
        ({'column_validation': True}, {'columns': None},
         'The param column_validation=True must use only, when there is ml.columns. Now ml.columns is None.'),
        ({'column_validation': True, 'custom_column_pydantic_model': 123}, {'columns': [{'name': '123'}]},
         'Only one of the two parameters can be used: column_validation, custom_column_pydantic_model. '
         'Now set column_validation=True, custom_column_pydantic_model=123.'),
    ]
)
def test_load_web_app_validation(print_model, web_app_params: Dict, ml_params: Dict, error: str):
    mlup_model = MLupModel(ml_model=print_model, conf=ModelConfig(**ml_params))
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(**web_app_params)
    )
    mlup_model.load()
    try:
        mlup_web_app.load()
        pytest.fail('Not raised error')
    except WebAppLoadError as e:
        assert str(e) == error


def test_load_web_app_with_not_loaded_ml_model(print_model):
    mlup_model = MLupModel(ml_model=print_model)
    mlup_web_app = MLupWebApp(ml=mlup_model)

    try:
        mlup_web_app.load()
        pytest.fail('Not raised error')
    except WebAppLoadError as e:
        assert str(e) == 'Model not loaded to memory. Analyze impossible. Please call ml.load().'


def test_run_web_app_with_not_loaded_ml_model(print_model):
    mlup_model = MLupModel(ml_model=print_model)
    mlup_web_app = MLupWebApp(ml=mlup_model)

    try:
        mlup_web_app.run()
        pytest.fail('Not raised error')
    except WebAppLoadError as e:
        assert str(e) == 'ML Model not loaded to memory. For run web app, please call ml.load(), or ml.load().'


def test_run_web_app_with_not_loaded_web_app(print_model):
    mlup_model = MLupModel(ml_model=print_model)
    mlup_web_app = MLupWebApp(ml=mlup_model)
    mlup_model.load()

    try:
        mlup_web_app.run()
        pytest.fail('Not raised error')
    except WebAppLoadError as e:
        assert str(e) == 'For run web app, please call web.load(), or web.load().'


@pytest.mark.parametrize(
    'debug, expected_json',
    [(False, DEFAULT_API_INFO_METHOD_DATA),  (True, DEFAULT_API_DEBUG_INFO_METHOD_DATA)],
    ids=['debug=False', 'debug=True']
)
@pytest.mark.asyncio
async def test_web_app_api_method_info(web_app_test_client, print_model, debug, expected_json):
    mlup_model = MLupModel(ml_model=print_model)
    mlup_web_app = MLupWebApp(ml=mlup_model, conf=WebAppConfig(debug=debug))
    mlup_model.load()
    mlup_web_app.load()

    with web_app_test_client(mlup_web_app) as api_test_client:
        response = await api_test_client.get("/info")
        assert response.status_code == 200
        assert response.json() == expected_json
        assertDictEqual(response.json(), expected_json)


@pytest.mark.asyncio
async def test_run_with_daemon_is_False(print_model):
    mlup_model = MLupModel(ml_model=print_model)
    mlup_web_app = MLupWebApp(ml=mlup_model, conf=WebAppConfig(host='0.0.0.0', port=8009))
    mlup_model.load()
    mlup_web_app.load()

    mlup_web_app.conf.uvicorn_kwargs['loop'] = 'none'

    web_app_thread = Thread(
        target=mlup_web_app.run,
        kwargs={'daemon': False},
        daemon=False,
        name='MLupWebAppDaemonTestsThread'
    )
    web_app_thread.start()
    await asyncio.sleep(0.5)

    try:
        async with httpx.AsyncClient() as client:
            response = await client.get('http://0.0.0.0:8009/health')
            assert response.status_code == 200
            assert response.json() == {'status': 200}
    finally:
        # Shutdown uvicorn not in main thread
        # https://stackoverflow.com/questions/58010119/are-there-any-better-ways-to-run-uvicorn-in-thread
        mlup_web_app._uvicorn_server.should_exit = True
        run_async(
            asyncio.wait_for,
            mlup_web_app._uvicorn_server.shutdown(),
            mlup_web_app.conf.timeout_for_shutdown_daemon
        )
        web_app_thread.join(timeout=3)


@pytest.mark.asyncio
async def test_run_with_daemon_is_True(print_model):
    mlup_model = MLupModel(ml_model=print_model)
    mlup_web_app = MLupWebApp(ml=mlup_model, conf=WebAppConfig(host='0.0.0.0', port=8011))
    mlup_model.load()
    mlup_web_app.load()

    mlup_web_app.run(daemon=True)
    await asyncio.sleep(0.5)

    try:
        async with httpx.AsyncClient() as client:
            response = await client.get('http://0.0.0.0:8011/health')
            assert response.status_code == 200
            assert response.json() == {'status': 200}
    finally:
        mlup_web_app.stop(shutdown_timeout=3)


@pytest.mark.asyncio
async def test_predict(web_app_test_client, print_model):
    mlup_model = MLupModel(
        ml_model=print_model,
        conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(mode=WebAppArchitecture.directly_to_predict)
    )
    mlup_model.load()
    mlup_web_app.load()
    with web_app_test_client(mlup_web_app) as api_test_client:
        response = await api_test_client.post("/predict", json={'X': [[1, 2, 3]]})
        assert response.status_code == 200
        assert response.headers['x-predict-id']
        assert response.json() == {"predict_result": [[1, 2, 3]]}


@pytest.mark.asyncio
async def test_http_health(web_app_test_client, print_model):
    mlup_model = MLupModel(
        ml_model=print_model,
        conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(mode=WebAppArchitecture.directly_to_predict)
    )
    mlup_model.load()
    mlup_web_app.load()
    with web_app_test_client(mlup_web_app) as api_test_client:
        response = await api_test_client.get("/health")
        assert response.status_code == 200
        assert response.json() == {"status": 200}


@pytest.mark.parametrize(
    'show_docs, expected_status',
    [(True, 200), (False, 404)],
    ids=['show_docs=True', 'show_docs=False']
)
@pytest.mark.asyncio
async def test_api_docs(web_app_test_client, print_model, show_docs, expected_status):
    mlup_model = MLupModel(
        ml_model=print_model,
        conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(mode=WebAppArchitecture.directly_to_predict, show_docs=show_docs)
    )
    mlup_model.load()
    mlup_web_app.load()
    with web_app_test_client(mlup_web_app) as api_test_client:
        response_docs = await api_test_client.get("/docs")
        response_redoc = await api_test_client.get("/redoc")
        assert response_docs.status_code == expected_status
        assert response_redoc.status_code == expected_status


@pytest.mark.asyncio
async def test_predict_model_numpy_returned_valid(web_app_test_client, list_to_numpy_array_model):
    mlup_model = MLupModel(
        ml_model=list_to_numpy_array_model,
        conf=ModelConfig(
            data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            data_transformer_for_predicted=ModelDataTransformerType.NUMPY_ARR,
        )
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(mode=WebAppArchitecture.directly_to_predict)
    )
    mlup_model.load()
    mlup_web_app.load()
    with web_app_test_client(mlup_web_app) as api_test_client:
        response = await api_test_client.post("/predict", json={'X': [[1, 2, 3]]})
        assert response.status_code == 200
        assert response.headers['x-predict-id']
        assert response.json() == {"predict_result": [[1, 2, 3]]}


@pytest.mark.asyncio
async def test_predict_model_numpy_returned_not_valid(web_app_test_client, list_to_numpy_array_model):
    mlup_model = MLupModel(
        ml_model=list_to_numpy_array_model,
        conf=ModelConfig(
            data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR,
            data_transformer_for_predicted=ModelDataTransformerType.PANDAS_DF,
        )
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(mode=WebAppArchitecture.directly_to_predict)
    )
    mlup_model.load()
    mlup_web_app.load()
    with web_app_test_client(mlup_web_app) as api_test_client:
        response = await api_test_client.post("/predict", json={'X': [[1, 2, 3]]})
        assert response.status_code == 500
        assert response.headers['x-predict-id']
        assert response.json() == {
            'detail': [
                {
                    'loc': [],
                    'msg': "'numpy.ndarray' object has no attribute 'to_dict'",
                    'type': 'predict_transform_data_error'
                }
            ],
            'predict_id': response.headers['x-predict-id']
        }


@pytest.mark.asyncio
async def test_predict_not_valid_request(web_app_test_client, print_model):
    mlup_model = MLupModel(
        ml_model=print_model,
        conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(mode=WebAppArchitecture.directly_to_predict)
    )
    mlup_model.load()
    mlup_web_app.load()
    with web_app_test_client(mlup_web_app) as api_test_client:
        response = await api_test_client.post("/predict", json={'not_exists_key': [[1, 2, 3]], 'test_param': 123})
        assert response.status_code == 422
        assert response.headers['x-predict-id']
        assert response.json() == {
            "detail": [
                {"loc": ["X"], "msg": "field required", "type": "value_error.missing"},
                {"loc": ["test_param"], "msg": "value could not be parsed to a boolean", "type": "type_error.bool"}
            ],
            "predict_id": response.headers['x-predict-id']
        }


@pytest.mark.asyncio
async def test_predict_max_requests_throttling(web_app_test_client, print_sleep_model):
    mlup_model = MLupModel(
        ml_model=print_sleep_model,
        conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(
            mode=WebAppArchitecture.directly_to_predict,
            throttling_max_requests=1,
        )
    )
    mlup_model.load()
    mlup_model.model_obj.sleep = 0.2
    mlup_web_app.load()
    with web_app_test_client(mlup_web_app) as api_test_client:
        first_request_task = create_async_task(
            api_test_client.post("/predict", json={'X': [[1, 2, 3]]}),
            name='test_predict_max_requests_throttling'
        )
        await asyncio.sleep(0.1)
        try:
            response = await api_test_client.post("/predict", json={'X': [[1, 2, 3]]})
            assert response.status_code == 429
            assert response.headers['x-predict-id']
            assert response.json() == {
                "detail": [
                    {"loc": [], "msg": "Max requests in app. Please try again later.", "type": "throttling_error"}
                ],
                "predict_id": response.headers['x-predict-id']
            }
        finally:
            if not first_request_task.cancelled():
                await first_request_task
                first_request_task.cancel()


@pytest.mark.asyncio
async def test_predict_max_request_len_throttling(web_app_test_client, print_model):
    mlup_model = MLupModel(
        ml_model=print_model,
        conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(
            mode=WebAppArchitecture.directly_to_predict,
            throttling_max_request_len=1,
        )
    )
    mlup_model.load()
    mlup_web_app.load()

    with web_app_test_client(mlup_web_app) as api_test_client:
        response = await api_test_client.post("/predict", json={'X': [[1, 2, 3], [3, 2, 1], [2, 1, 3]]})
        assert response.status_code == 429
        assert response.headers['x-predict-id']
        assert response.json() == {
            "detail": [
                {
                    "loc": [],
                    "msg": "The query exceeded the limit on the number of rows for the predict. "
                           "Please downsize your request.",
                    "type": "throttling_error"
                }
            ],
            "predict_id": response.headers['x-predict-id']
        }


@pytest.mark.asyncio
async def test_predict_transform_error(web_app_test_client, print_model):
    mlup_model = MLupModel(
        ml_model=print_model,
        conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.PANDAS_DF),
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(mode=WebAppArchitecture.directly_to_predict),
    )
    mlup_model.load()
    mlup_web_app.load()

    with web_app_test_client(mlup_web_app) as api_test_client:
        response = await api_test_client.post("/predict", json={'X': [[1, 2, 3]]})
        assert response.status_code == 500
        assert response.headers['x-predict-id']
        assert response.json() == {
            "detail": [
                {
                    "loc": [],
                    "msg": "If input data have type list, than need set columns.",
                    "type": "predict_transform_data_error"
                }
            ],
            "predict_id": response.headers['x-predict-id']
        }


@pytest.mark.asyncio
async def test_predict_custom_model_error(web_app_test_client, raise_exception_model):
    mlup_model = MLupModel(
        ml_model=raise_exception_model,
        conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(mode=WebAppArchitecture.directly_to_predict)
    )
    mlup_model.load()
    mlup_web_app.load()

    with web_app_test_client(mlup_web_app) as api_test_client:
        response = await api_test_client.post("/predict", json={'X': [[1, 2, 3]]})
        assert response.status_code == 500
        assert response.headers['x-predict-id']
        assert response.json() == {
            'detail': [{'loc': [], 'msg': 'Test message', 'type': 'predict_error'}],
            'predict_id': response.headers['x-predict-id']
        }


@pytest.mark.asyncio
async def test_get_predict_result(web_app_test_client, print_sleep_model):
    mlup_model = MLupModel(
        ml_model=print_sleep_model,
        conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
    )
    mlup_web_app = MLupWebApp(
        ml=mlup_model,
        conf=WebAppConfig(
            mode=WebAppArchitecture.worker_and_queue,
            is_long_predict=True,
            ttl_client_wait=0.2,
        )
    )
    mlup_model.load()
    mlup_model.model_obj.sleep = 0.5
    mlup_web_app.load()

    with web_app_test_client(mlup_web_app) as api_test_client:
        response = await api_test_client.post("/predict", json={'X': [[1, 2, 3]]})
        assert response.status_code == 200
        assert response.headers['x-predict-id']
        predict_id = response.json()['predict_result']['predict_id']
        assert predict_id

        # Check, model working yet
        response = await api_test_client.get(f'/get-predict/{predict_id}')
        assert response.status_code == 408
        assert response.json() == {
            'detail': [
                {'loc': [], 'msg': 'Response timed out. Repeat request please.', 'type': 'predict_wait_result_error'}
            ],
            'predict_id': predict_id
        }

        # Mock response architecture
        mlup_web_app._architecture_obj.ttl_client_wait = 30
        mlup_web_app._architecture_obj.results_storage[predict_id] = ([[1, 2, 3]], None)

        # Check get results
        response = await api_test_client.get(f'/get-predict/{predict_id}')
        assert response.status_code == 200
        assert response.json() == [[1, 2, 3]]

tests/unit_tests/web/test_app.py
nxexox/pymlup
import asyncio
from asyncio import InvalidStateError
import random
import time

import pytest
from fastapi import FastAPI

from mlup.constants import ModelDataTransformerType, WebAppArchitecture
from mlup.errors import ModelLoadError, PredictError, WebAppLoadError, PredictWaitResultError
from mlup.ml.model import MLupModel, ModelConfig
from mlup.utils.collections import TTLOrderedDict
from mlup.utils.interspection import get_class_by_path
from mlup.utils.loop import create_async_task
from mlup.web.architecture.directly_to_predict import DirectlyToPredictArchitecture
from mlup.web.architecture.worker_and_queue import WorkerAndQueueArchitecture
from mlup.web.architecture.batching import BatchingSingleProcessArchitecture


test_predict_id = 'test_predict_id'


def shuffle_array(arr):
    return sorted(arr, key=lambda x: random.randint(0, 10))


@pytest.mark.parametrize(
    'archi_type, expected_class', [
        (WebAppArchitecture.directly_to_predict, DirectlyToPredictArchitecture),
        (WebAppArchitecture.worker_and_queue, WorkerAndQueueArchitecture),
        (WebAppArchitecture.batching, BatchingSingleProcessArchitecture),
    ]
)
def test_get_architecture_by_type(archi_type, expected_class):
    assert get_class_by_path(archi_type) == expected_class


def test_get_architecture_by_type_bad_type():
    try:
        get_class_by_path('not exists architecture type')
        pytest.fail('Not raised KeyError with not exists type')
    except ModuleNotFoundError as e:
        assert str(e) == "No module named 'not exists architecture type'"


def test_get_architecture_by_custom_type():
    data = get_class_by_path('mlup.constants.ModelDataTransformerType')
    assert issubclass(data, ModelDataTransformerType)


class TestDirectlyToPredictArchitecture:
    def test_create_with_empty_params(self):
        try:
            DirectlyToPredictArchitecture()
        except KeyError as e:
            assert str(e) == "'ml'"

    def test_create(self, print_model):
        archi = DirectlyToPredictArchitecture(ml=MLupModel(ml_model=print_model), test_value=123)
        assert archi.extra == dict(test_value=123)

    def test_load(self, print_model):
        archi = DirectlyToPredictArchitecture(ml=MLupModel(ml_model=print_model))
        archi.load()

    @pytest.mark.asyncio
    async def test_predict(self, print_model):
        mlup_model = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        mlup_model.load()
        archi = DirectlyToPredictArchitecture(ml=mlup_model)
        archi.load()
        pred_data = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
        assert pred_data == [[1, 2, 3]]

    @pytest.mark.asyncio
    async def test_predict_error(self, raise_exception_model):
        mlup_model = MLupModel(
            ml_model=raise_exception_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        mlup_model.load()
        archi = DirectlyToPredictArchitecture(ml=mlup_model)
        archi.load()
        try:
            await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            pytest.fail('Not raised error')
        except PredictError as e:
            assert str(e) == 'Test message'

    @pytest.mark.asyncio
    async def test_predict_with_not_loaded_model(self, print_model):
        mlup_model = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        archi = DirectlyToPredictArchitecture(ml=mlup_model)
        archi.load()
        try:
            await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            pytest.fail('Not raised error')
        except ModelLoadError as e:
            assert str(e) == 'Model object not loaded. Please call load().'

    @pytest.mark.asyncio
    async def test_predict_with_not_loaded_archi(self, print_model):
        mlup_model = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        mlup_model.load()
        archi = DirectlyToPredictArchitecture(ml=mlup_model)
        archi.load()
        pred_data = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
        assert pred_data == [[1, 2, 3]]


class TestWorkerAndQueueArchitecture:
    _default_params = dict(
        item_id_col_name='col_name',
        max_queue_size=5,
        ttl_predicted_data=13,
        is_long_predict=True,
        ttl_client_wait=30.0,
        # additional values
        additional_value=123,
    )

    def test_create(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **self._default_params
        )
        assert archi.fast_api == fast_api
        assert archi.ml == ml
        assert archi.item_id_col_name == 'col_name'
        assert archi.max_queue_size == 5
        assert archi.ttl_predicted_data == 13
        assert archi.is_long_predict is True
        assert archi.extra == dict(additional_value=123)

    def test_create_with_empty_params(self):
        try:
            WorkerAndQueueArchitecture()
            pytest.fail('Error not raised')
        except KeyError as e:
            assert str(e) == "'fast_api'"

    @pytest.mark.parametrize(
        'ttl_predicted_data',
        [0, -1]
    )
    def test_create_with_ttl_predicted_data_less_0(self, print_model, ttl_predicted_data):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        params = self._default_params.copy()
        params['ttl_predicted_data'] = ttl_predicted_data
        try:
            WorkerAndQueueArchitecture(
                fast_api=fast_api,
                ml=ml,
                **params
            )
        except ValueError as e:
            assert str(e) == 'The param ttl_predicted_data parameter must be greater than 0. ' \
                             f'Now it is {ttl_predicted_data}'

    def test_load(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **self._default_params,
        )
        assert archi.results_storage is None
        assert archi.queries_queue is None
        archi.load()
        assert isinstance(archi.results_storage, TTLOrderedDict)
        assert isinstance(archi.queries_queue, asyncio.Queue)

    def test_reload(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **self._default_params,
        )
        assert archi.results_storage is None
        assert archi.queries_queue is None
        archi.load()
        assert isinstance(archi.results_storage, TTLOrderedDict)
        assert isinstance(archi.queries_queue, asyncio.Queue)
        archi.results_storage['test_key'] = 'test_value'
        assert archi.queries_queue.qsize() == 0
        archi.queries_queue.put_nowait('test_value')
        assert archi.queries_queue.qsize() == 1

        archi.load()
        assert isinstance(archi.results_storage, TTLOrderedDict)
        assert isinstance(archi.queries_queue, asyncio.Queue)
        assert 'test_key' not in archi.results_storage
        assert archi.queries_queue.qsize() == 0

    def test_load_is_long_predict_is_True(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        params = self._default_params.copy()
        params['is_long_predict'] = True
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        assert any(r.path == '/get-predict/{predict_id}' for r in archi.fast_api.routes)

    def test_load_is_long_predict_is_False(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        params = self._default_params.copy()
        params['is_long_predict'] = False
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        assert all(r.path != '/get-predict/{predict_id}' for r in archi.fast_api.routes)

    @pytest.mark.asyncio
    async def test_run_with_not_loaded(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **self._default_params,
        )
        try:
            await archi.run()
            pytest.fail('Error not raised')
        except WebAppLoadError as e:
            assert str(e) == 'Not called .load() in web.load()'

    @pytest.mark.asyncio
    async def test_run(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        params = self._default_params.copy()
        params['is_long_predict'] = True
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.1)
        try:
            assert archi.is_running
            try:
                await archi.run()
                pytest.fail('Error not raised')
            except WebAppLoadError as e:
                assert str(e) == 'Worker is already running'
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_stop(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        params = self._default_params.copy()
        params['is_long_predict'] = True
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.1)
        assert archi.is_running
        await archi.stop()
        assert archi.is_running is False

    @pytest.mark.asyncio
    async def test_predict(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['is_long_predict'] = False
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            pred_result = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            assert pred_result == [[1, 2, 3]]
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_predict_with_little_max_queue_size(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['max_queue_size'] = 1
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id + '2')
            pytest.fail('Not raised error')
        except PredictError as e:
            assert str(e) == 'Queue is full. Please try later.'
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_predict_with_little_ttl_predicted_data(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['is_long_predict'] = True
        params['ttl_predicted_data'] = 1
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            pred_id_1 = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            await asyncio.sleep(1)
            pred_id_2 = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id + '2')
            pred_result_2 = await archi.get_predict_result(pred_id_2["predict_id"])
            assert pred_result_2 == [[1, 2, 3]]

            # Not found results
            task = create_async_task(
                archi.get_predict_result(pred_id_1["predict_id"]),
                name='test_predict_with_little_ttl_predicted_data',
            )
            await asyncio.sleep(0.2)
            if task.cancelled():
                pytest.fail('Tasks could not have status cancelled.')
            task.cancel()
            task.result()
            pytest.fail('Not raised error')
        except InvalidStateError as e:
            assert str(e) == 'Result is not set.'
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_predict_with_little_ttl_client_wait(self, print_sleep_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_sleep_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        ml.model_obj.sleep = 0.3
        params = self._default_params.copy()
        params['is_long_predict'] = False
        params['ttl_client_wait'] = 0.2
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            pytest.fail('Not raised error')
        except PredictWaitResultError as e:
            assert str(e) == 'Response timed out. Repeat request please.'
            assert e.predict_id
            await asyncio.sleep(0.2)
            pred_result = await archi.get_predict_result(e.predict_id)
            assert pred_result == [[1, 2, 3]]
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_predict_without_run(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['is_long_predict'] = False
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        try:
            await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            pytest.fail('Error not raised')
        except PredictError as e:
            assert str(e) == 'Worker not started. Please call web.run()'

    @pytest.mark.asyncio
    async def test_get_result_predict(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['is_long_predict'] = True
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            pred_id_1 = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            pred_id_2 = await archi.predict(data_for_predict={'X': [[3, 2, 1]]}, predict_id=test_predict_id + '2')
            pred_1 = await archi.get_predict_result(pred_id_1['predict_id'])
            pred_2 = await archi.get_predict_result(pred_id_2['predict_id'])
            assert pred_1 == [[1, 2, 3]]
            assert pred_2 == [[3, 2, 1]]
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_get_result_predict_with_little_ttl_client_wait(self, print_sleep_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_sleep_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        ml.model_obj.sleep = 0.3
        params = self._default_params.copy()
        params['is_long_predict'] = True
        params['ttl_client_wait'] = 0.2
        archi = WorkerAndQueueArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            pred_id_1 = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            try:
                await archi.get_predict_result(pred_id_1['predict_id'])
                pytest.fail('Not raised error')
            except PredictWaitResultError as e:
                assert str(e) == 'Response timed out. Repeat request please.'
                assert pred_id_1['predict_id'] == e.predict_id
        finally:
            await archi.stop()


class TestBatchingSingleProcessArchitecture:
    _default_params = dict(
        item_id_col_name='col_name',
        min_batch_len=10,
        batch_worker_timeout=1.0,
        max_queue_size=5,
        ttl_predicted_data=13,
        is_long_predict=True,
        ttl_client_wait=30.0,
        # additional values
        additional_value=123,
    )

    def test_create(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **self._default_params
        )
        assert archi.fast_api == fast_api
        assert archi.ml == ml
        assert archi.item_id_col_name == 'col_name'
        assert archi.min_batch_len == 10
        assert archi.batch_worker_timeout == 1.0
        assert archi.max_queue_size == 5
        assert archi.ttl_predicted_data == 13
        assert archi.is_long_predict is True
        assert archi.extra == dict(additional_value=123)

    def test_create_with_empty_params(self):
        try:
            BatchingSingleProcessArchitecture()
            pytest.fail('Error not raised')
        except KeyError as e:
            assert str(e) == "'fast_api'"

    @pytest.mark.parametrize(
        'param_name, param_value',
        [
            ('batch_worker_timeout', -1),
            ('batch_worker_timeout', 0),
            ('ttl_predicted_data', -1),
            ('ttl_predicted_data', 0)
        ]
    )
    def test_create_with_param_less_0(self, print_model, param_name, param_value):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        params = self._default_params.copy()
        params[param_name] = param_value
        try:
            BatchingSingleProcessArchitecture(
                fast_api=fast_api,
                ml=ml,
                **params
            )
        except ValueError as e:
            assert str(e) == f'The param {param_name} parameter must be greater than 0. ' \
                             f'Now it is {param_value}'

    def test_load(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **self._default_params,
        )
        assert archi.results_storage is None
        assert archi.queries_queue is None
        archi.load()
        assert isinstance(archi.results_storage, TTLOrderedDict)
        assert isinstance(archi.queries_queue, asyncio.Queue)

    def test_reload(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **self._default_params,
        )
        assert archi.results_storage is None
        assert archi.queries_queue is None
        archi.load()
        assert isinstance(archi.results_storage, TTLOrderedDict)
        assert isinstance(archi.queries_queue, asyncio.Queue)
        archi.results_storage['test_key'] = 'test_value'
        assert archi.queries_queue.qsize() == 0
        archi.queries_queue.put_nowait('test_value')
        assert archi.queries_queue.qsize() == 1

        archi.load()
        assert isinstance(archi.results_storage, TTLOrderedDict)
        assert isinstance(archi.queries_queue, asyncio.Queue)
        assert 'test_key' not in archi.results_storage
        assert archi.queries_queue.qsize() == 0

    def test_load_is_long_predict_is_True(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        params = self._default_params.copy()
        params['is_long_predict'] = True
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        assert any(r.path == '/get-predict/{predict_id}' for r in archi.fast_api.routes)

    def test_load_is_long_predict_is_False(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        params = self._default_params.copy()
        params['is_long_predict'] = False
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        assert all(r.path != '/get-predict/{predict_id}' for r in archi.fast_api.routes)

    @pytest.mark.asyncio
    async def test_run_with_not_loaded(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **self._default_params,
        )
        try:
            await archi.run()
            pytest.fail('Error not raised')
        except WebAppLoadError as e:
            assert str(e) == 'Not called .load() in web.load()'

    @pytest.mark.asyncio
    async def test_run(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **self._default_params,
        )
        archi.load()
        await archi.run()
        try:
            await asyncio.sleep(0.1)
            assert archi.is_running
            try:
                await archi.run()
                pytest.fail('Error not raised')
            except WebAppLoadError as e:
                assert str(e) == 'Worker is already running'
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_stop(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(ml_model=print_model)
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **self._default_params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.1)
        assert archi.is_running
        await archi.stop()
        assert archi.is_running is False

    @pytest.mark.asyncio
    async def test_predict(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['is_long_predict'] = False
        params['batch_worker_timeout'] = 0.1
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            pred_result = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            assert pred_result == [[1, 2, 3]]
        finally:
            await archi.stop()

    @pytest.mark.parametrize('min_batch_len', [2, 5, 10, 50])
    @pytest.mark.asyncio
    async def test_predict_with_min_batch_len(self, print_model, min_batch_len):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['is_long_predict'] = True
        params['min_batch_len'] = min_batch_len
        params['batch_worker_timeout'] = 100
        params['ttl_client_wait'] = 0.5
        params['max_queue_size'] = 100
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)

        try:
            arr = list(range(10))
            arrs = [shuffle_array(arr)]

            pred_id_1 = await archi.predict(data_for_predict={'X': [arrs[0]]}, predict_id=test_predict_id)
            try:
                await archi.get_predict_result(pred_id_1['predict_id'])
                pytest.fail('Not raised error')
            except PredictWaitResultError as e:
                assert str(e) == 'Response timed out. Repeat request please.'
                assert e.predict_id == pred_id_1['predict_id']
            pred_ids = [pred_id_1]
            for i in range(1, min_batch_len):
                arrs.append(shuffle_array(arr))
                _pred_id = await archi.predict(data_for_predict={'X': [arrs[-1]]}, predict_id=test_predict_id + str(i))
                pred_ids.append(_pred_id)
            for i, pred_id in enumerate(pred_ids):
                pred_res = await archi.get_predict_result(pred_id['predict_id'])
                assert pred_res == [arrs[i]]
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_predict_with_batch_worker_timeout(self, print_sleep_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_sleep_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        ml.model_obj.sleep = 0.1
        params = self._default_params.copy()
        params['is_long_predict'] = True
        params['ttl_predicted_data'] = 1
        params['min_batch_len'] = 100
        params['batch_worker_timeout'] = 0.001
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.02)
        try:
            start = time.monotonic()
            pred_id_1 = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            await asyncio.sleep(0.01)
            pred_id_2 = await archi.predict(data_for_predict={'X': [[3, 2, 1]]}, predict_id=test_predict_id + '1')
            assert len(archi.batch_queue) == 1
            assert archi.queries_queue.qsize() == 1

            pred_result_1 = await archi.get_predict_result(pred_id_1["predict_id"])
            start_two = time.monotonic()
            pred_result_2 = await archi.get_predict_result(pred_id_2["predict_id"])
            end = time.monotonic()
            assert pred_result_1 == [[1, 2, 3]]
            assert pred_result_2 == [[3, 2, 1]]
            assert end - start < 0.4
            assert end - start_two < 0.25
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_predict_with_little_max_queue_size(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['is_long_predict'] = True
        params['max_queue_size'] = 1
        params['batch_worker_timeout'] = 0.1
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id + '1')
            pytest.fail('Not raised error')
        except PredictError as e:
            assert str(e) == 'Queue is full. Please try later.'
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_predict_with_little_ttl_predicted_data(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['is_long_predict'] = True
        params['ttl_predicted_data'] = 1
        params['batch_worker_timeout'] = 0.1
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            pred_id_1 = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            await asyncio.sleep(1)
            pred_id_2 = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id + '1')
            pred_result_2 = await archi.get_predict_result(pred_id_2["predict_id"])
            assert pred_result_2 == [[1, 2, 3]]

            # Not found results
            task = create_async_task(
                archi.get_predict_result(pred_id_1["predict_id"]),
                name='test_predict_with_little_ttl_predicted_data'
            )
            await asyncio.sleep(0.2)
            if task.cancelled():
                pytest.fail('Tasks could not have status cancelled.')
            task.cancel()
            task.result()
            pytest.fail('Not raised error')
        except InvalidStateError as e:
            assert str(e) == 'Result is not set.'
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_predict_with_little_ttl_client_wait(self, print_sleep_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_sleep_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        ml.model_obj.sleep = 0.3
        params = self._default_params.copy()
        params['is_long_predict'] = False
        params['ttl_client_wait'] = 0.3
        params['batch_worker_timeout'] = 0.01
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            pytest.fail('Not raised error')
        except PredictWaitResultError as e:
            assert str(e) == 'Response timed out. Repeat request please.'
            assert e.predict_id
            await asyncio.sleep(0.2)
            pred_result = await archi.get_predict_result(e.predict_id)
            assert pred_result == [[1, 2, 3]]
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_predict_without_run(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['is_long_predict'] = False
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        try:
            await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            pytest.fail('Error not raised')
        except PredictError as e:
            assert str(e) == 'Worker not started. Please call web.run()'

    @pytest.mark.asyncio
    async def test_get_result_predict(self, print_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        params = self._default_params.copy()
        params['is_long_predict'] = True
        params['batch_worker_timeout'] = 0.1
        params['ttl_client_wait'] = 1.0
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.05)
        try:
            pred_id_1 = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            pred_id_2 = await archi.predict(data_for_predict={'X': [[3, 2, 1]]}, predict_id=test_predict_id + '1')
            pred_1 = await archi.get_predict_result(pred_id_1['predict_id'])
            pred_2 = await archi.get_predict_result(pred_id_2['predict_id'])
            assert pred_1 == [[1, 2, 3]]
            assert pred_2 == [[3, 2, 1]]
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_get_result_predict_with_little_ttl_client_wait(self, print_sleep_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_sleep_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        ml.model_obj.sleep = 0.3
        params = self._default_params.copy()
        params['is_long_predict'] = True
        params['ttl_client_wait'] = 0.2
        params['batch_worker_timeout'] = 0.1
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.02)
        try:
            pred_id_1 = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            try:
                await archi.get_predict_result(pred_id_1['predict_id'])
                pytest.fail('Not raised error')
            except PredictWaitResultError as e:
                assert str(e) == 'Response timed out. Repeat request please.'
                assert pred_id_1['predict_id'] == e.predict_id
        finally:
            await archi.stop()

    @pytest.mark.asyncio
    async def test_predict_with_many_thick_requests(self, print_sleep_model):
        fast_api = FastAPI()
        ml = MLupModel(
            ml_model=print_sleep_model,
            conf=ModelConfig(data_transformer_for_predict=ModelDataTransformerType.NUMPY_ARR)
        )
        ml.load()
        ml.model_obj.sleep = 0.1
        params = self._default_params.copy()
        params['is_long_predict'] = True
        params['ttl_predicted_data'] = 10
        params['ttl_client_wait'] = 1
        params['max_queue_size'] = 100
        params['min_batch_len'] = 10
        params['batch_worker_timeout'] = 0.3
        archi = BatchingSingleProcessArchitecture(
            fast_api=fast_api,
            ml=ml,
            **params,
        )
        archi.load()
        await archi.run()
        await asyncio.sleep(0.02)
        try:
            pred_id_1 = await archi.predict(data_for_predict={'X': [[1, 2, 3]]}, predict_id=test_predict_id)
            pred_id_9 = await archi.predict(
                data_for_predict={'X': [[3, 2, 1] for _ in range(9)]},
                predict_id=test_predict_id + '9'
            )
            pred_id_20 = await archi.predict(
                data_for_predict={'X': [[2, 3, 1] for _ in range(20)]},
                predict_id=test_predict_id + '20'
            )

            pred_res_1 = await archi.get_predict_result(pred_id_1['predict_id'])
            pred_res_9 = await archi.get_predict_result(pred_id_9['predict_id'])
            pred_res_20 = await archi.get_predict_result(pred_id_20['predict_id'])
            assert pred_res_20 == [[2, 3, 1] for _ in range(20)]
            assert pred_res_1 == [[1, 2, 3]]
            assert pred_res_9 == [[3, 2, 1] for _ in range(9)]
        finally:
            await archi.stop()

tests/unit_tests/web/test_architecture.py
nxexox/pymlup
from mlup.constants import ITEM_ID_COL_NAME, WebAppArchitecture
from mlup.web.app import WebAppConfig


def test_model_config_default_values():
    conf = WebAppConfig()
    assert conf.host == '0.0.0.0'
    assert conf.port == 8009
    assert conf.web_app_version == '1.0.0.0'
    assert conf.column_validation is False
    assert conf.custom_column_pydantic_model is None
    assert conf.mode == WebAppArchitecture.directly_to_predict
    assert conf.max_queue_size == 100
    assert conf.ttl_predicted_data == 60
    assert conf.ttl_client_wait == 30.0
    assert conf.min_batch_len == 10
    assert conf.batch_worker_timeout == 1.0
    assert conf.is_long_predict is False
    assert conf.throttling_max_requests is None
    assert conf.throttling_max_request_len is None
    assert conf.timeout_for_shutdown_daemon == 3.0
    assert conf.item_id_col_name == ITEM_ID_COL_NAME

tests/unit_tests/web/test_web_app_config.py
nxexox/pymlup