Web-based platform for media-centric (video, audio and images) subjective testing
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.

import os
import sys

import django

sys.path.insert(0, os.path.abspath('../..'))

django.setup()

# -- Project information -----------------------------------------------------

project = 'NEST'
copyright = '2020, Netflix'
author = 'zli, lkrasula'

# The full version, including alpha/beta/rc tags
release = '0.0.1'


# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'rinoh.frontend.sphinx',
    'sphinx.ext.todo',
    'sphinx.ext.viewcode',
    'sphinx.ext.autodoc',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []


# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    'preamble': '',

    # Latex figure (float) alignment
    'figure_align': 'htbp',
    }

nest_site/docs/source/conf.py
Netflix/e2nest
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys


def main():
    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nest_site.settings')
    try:
        from django.core.management import execute_from_command_line
    except ImportError as exc:
        raise ImportError(
            "Couldn't import Django. Are you sure it's installed and "
            "available on your PYTHONPATH environment variable? Did you "
            "forget to activate a virtual environment?"
        ) from exc
    execute_from_command_line(sys.argv)


if __name__ == '__main__':
    main()

nest_site/manage.py
Netflix/e2nest
from nest.sites import site

__all__ = ['site']

nest_site/nest/__init__.py
Netflix/e2nest
from django.contrib import admin
from django.urls import reverse
from django.utils.encoding import force_str
from django.utils.safestring import mark_safe

from .models import Experiment, Experimenter, Round, \
    Session, Stimulus, StimulusGroup, StimulusVoteGroup, Subject, Vote, VoteRegister


# =================
# ==== Inline =====
# =================


class SessionInline(admin.StackedInline):
    model = Session
    extra = 0
    fields = ['experiment', 'subject',
              'get_edit_link',
              # 'is_active',
              ]
    readonly_fields = ['experiment',
                       # 'subject',
                       'get_edit_link',
                       # 'is_active',
                       ]

    def get_edit_link(self, obj=None):
        if obj.pk:
            url = reverse('admin:%s_%s_change' % (obj._meta.app_label,
                                                  obj._meta.model_name),
                          args=[force_str(obj.pk)])
            return mark_safe("""<a href="{url}">{text}</a>""".format(
                url=url,
                text="Edit this %s separately" % obj._meta.verbose_name,
            ))
        return "(save and continue editing to create a link)"

    get_edit_link.short_description = "Edit link"


class RoundInline(admin.StackedInline):
    model = Round
    extra = 0
    fields = ['session',
              'get_edit_link',
              'response_sec',
              # 'is_active',
              ]
    readonly_fields = ['session',
                       'get_edit_link',
                       'response_sec',
                       # 'is_active',
                       ]

    def get_edit_link(self, obj=None):
        if obj.pk:
            url = reverse('admin:%s_%s_change' % (obj._meta.app_label,
                                                  obj._meta.model_name),
                          args=[force_str(obj.pk)])
            return mark_safe("""<a href="{url}">{text}</a>""".format(
                url=url,
                text="Edit this %s separately" % obj._meta.verbose_name,
            ))
        return "(save and continue editing to create a link)"

    get_edit_link.short_description = "Edit link"


class VoteInline(admin.StackedInline):
    model = Vote
    extra = 0
    fields = ['score', 'round',
              'get_edit_link',
              # 'is_active',
              ]
    readonly_fields = ['score', 'round',
                       'get_edit_link',
                       # 'is_active',
                       ]

    def get_edit_link(self, obj=None):
        if obj.pk:
            url = reverse('admin:%s_%s_change' % (obj._meta.app_label,
                                                  'vote'),
                          args=[force_str(obj.pk)])
            return mark_safe("""<a href="{url}">{text}</a>""".format(
                url=url,
                text="Edit this %s separately" % 'vote',
            ))
        return "(save and continue editing to create a link)"

    get_edit_link.short_description = "Edit link"


# ====================
# ==== ModelAdmin ====
# ====================


class ExperimentAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['title', 'id',
                                         'experimenter_list',
                                         'description']}),
        ('Date information', {'fields': ['create_date',
                                         # 'is_active',
                                         # 'deactivate_date',
                                         ]}),
        ('Tools', {'fields': ['download_dataset']}),
    ]
    readonly_fields = ['id',
                       'create_date',
                       # 'is_active',
                       'download_dataset',
                       'experimenter_list',
                       ]
    inlines = [SessionInline]
    list_display = ('title', 'id', 'experimenter_list', 'description', 'create_date',
                    'download_dataset')
    list_filter = ['create_date',
                   # 'deactivate_date',
                   ]
    search_fields = ['title', 'description']

    def download_dataset(self, obj):
        e = Experiment.objects.get(id=obj.id)
        return mark_safe(
            """<a href="{sureal}">S1</a> | <a href="{sureal_alt}">S2</a> | <a href="{nest}">NEST</a> | <a href="{nest_csv}">CSV</a>""".
            format(
                sureal=reverse('admin:download_sureal', args=[force_str(e.pk)]),
                sureal_alt=reverse('admin:download_sureal_alt', args=[force_str(e.pk)]),
                nest=reverse('admin:download_nest', args=[force_str(e.pk)]),
                nest_csv=reverse('admin:download_nest_csv', args=[force_str(e.pk)])))

    download_dataset.short_description = 'Dataset'

    def experimenter_list(self, obj):
        er_html_list = list()
        for er in obj.experimenters.all():
            text = str(er)
            url = reverse('admin:%s_%s_change' % (er._meta.app_label,
                                                  er._meta.model_name),
                          args=[force_str(er.pk)])
            er_html_list.append("""<a href="{url}">{text}</a>""".format(
                url=url, text=text))
        return mark_safe("<br>".join(er_html_list))

    experimenter_list.short_description = 'Experimenters'


class SessionAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['__str__', 'id', 'experiment', 'subject']}),
        ('Date information', {'fields': ['create_date',
                                         # 'is_active',
                                         # 'deactivate_date',
                                         ]}),
    ]
    readonly_fields = ['__str__', 'id',
                       'create_date',
                       'experiment',
                       # 'subject',
                       # 'is_active',
                       ]
    inlines = [RoundInline]
    list_display = ('__str__', 'id',
                    'experiment',
                    'subject',
                    # 'is_active',
                    )
    list_filter = ['create_date',
                   # 'deactivate_date',
                   ]


class RoundAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['__str__', 'id',
                                         'round_id',
                                         'session',
                                         'stimulusgroup',
                                         'response_sec',
                                         ]}),
        ('Date information', {'fields': ['create_date',
                                         # 'is_active',
                                         # 'deactivate_date',
                                         ]}),
    ]
    readonly_fields = ['__str__', 'id',
                       'round_id',
                       'create_date',
                       'session',
                       'stimulusgroup',
                       'response_sec',
                       # 'is_active',
                       ]
    inlines = [VoteInline]
    list_display = ('__str__', 'id',
                    'round_id',
                    'session',
                    'stimulusgroup',
                    'response_sec',
                    # 'is_active',
                    )
    list_filter = ['create_date',
                   # 'deactivate_date',
                   ]


class VoteAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['__str__', 'id',
                                         'score', 'round', 'stimulusvotegroup']}),
        ('Date information', {'fields': ['create_date',
                                         # 'is_active',
                                         # 'deactivate_date',
                                         ]}),
    ]
    readonly_fields = ['__str__', 'id',
                       'create_date',
                       # 'is_active',
                       'round', 'stimulusvotegroup']
    list_display = ('__str__', 'id',
                    'score',
                    'round',
                    # 'is_active',
                    )
    list_filter = ['create_date',
                   # 'deactivate_date',
                   ]


class StimulusAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['__str__', 'id',
                                         'stimulus_id',
                                         'experiment',
                                         'content',
                                         'condition']}),
        ('Date information', {'fields': ['create_date',
                                         # 'is_active',
                                         # 'deactivate_date',
                                         ]}),
    ]
    readonly_fields = ['__str__', 'id',
                       'create_date',
                       'stimulus_id',
                       'experiment',
                       'content',
                       'condition',
                       # 'is_active',
                       ]
    list_display = ('__str__', 'id',
                    'stimulus_id',
                    'experiment',
                    'content', 'condition',
                    # 'is_active',
                    )
    list_filter = ['create_date',
                   # 'deactivate_date',
                   ]


class StimulusGroupAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['__str__', 'id',
                                         'stimulusgroup_id',
                                         'experiment',
                                         'stimuli']}),
        ('Date information', {'fields': ['create_date',
                                         # 'is_active',
                                         # 'deactivate_date',
                                         ]}),
    ]
    readonly_fields = ['create_date',
                       '__str__', 'id',
                       'stimulusgroup_id',
                       'experiment',
                       'stimuli',
                       # 'is_active',
                       ]
    list_display = ('__str__',
                    'id',
                    'stimulusgroup_id',
                    'experiment',
                    'stimuli',
                    # 'is_active',
                    )
    list_filter = ['create_date',
                   # 'deactivate_date',
                   ]


class StimulusVoteGroupAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['__str__', 'id',
                                         'stimulus_list',
                                         'stimulusgroup',
                                         'stimulusvotegroup_id',
                                         'experiment',
                                         ]}),
        ('Date information', {'fields': ['create_date',
                                         # 'is_active',
                                         # 'deactivate_date',
                                         ]}),
    ]
    readonly_fields = ['create_date',
                       'stimulus_list',
                       'stimulusgroup',
                       'stimulusvotegroup_id',
                       'experiment',
                       '__str__', 'id',
                       # 'is_active',
                       ]
    list_display = ('__str__', 'id', 'stimulus_list', 'stimulusgroup',
                    'stimulusvotegroup_id',
                    'experiment',
                    # 'is_active',
                    )
    list_filter = ['create_date',
                   # 'deactivate_date',
                   ]

    def stimulus_list(self, obj):
        stim_html_list = list()
        vrs = VoteRegister.objects.filter(stimulusvotegroup=obj)
        for vr in vrs:
            stimulus_order = vr.stimulus_order
            s = vr.stimulus
            text = str(s) + f' [order {stimulus_order}]'
            url = reverse('admin:%s_%s_change' % (s._meta.app_label,
                                                  s._meta.model_name),
                          args=[force_str(s.pk)])
            stim_html_list.append("""<a href="{url}">{text}</a>""".format(
                url=url, text=text))

        return mark_safe("<br>".join(stim_html_list))

    stimulus_list.short_description = 'Stimulus List'


class SubjectAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['__str__', 'id',
                                         'name', 'user']}),
        ('Date information', {'fields': ['create_date',
                                         # 'is_active',
                                         # 'deactivate_date',
                                         ]}),
    ]
    readonly_fields = ['__str__', 'id',
                       'create_date',
                       'name', 'user',
                       # 'is_active',
                       ]
    inlines = [SessionInline]
    list_display = ('__str__', 'id',
                    'name', 'user',
                    # 'is_active',
                    )
    list_filter = ['create_date',
                   # 'deactivate_date',
                   ]
    search_fields = ['name']


class ExperimenterAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['__str__', 'id',
                                         'name', 'user']}),
        ('Date information', {'fields': ['create_date',
                                         # 'is_active',
                                         # 'deactivate_date',
                                         ]}),
    ]
    readonly_fields = ['__str__', 'id',
                       'create_date',
                       'name', 'user',
                       # 'is_active',
                       ]
    list_display = ('__str__', 'id',
                    'name', 'user',
                    # 'is_active',
                    )
    list_filter = ['create_date',
                   # 'deactivate_date',
                   ]
    search_fields = ['name']


# ==================
# ==== register ====
# ==================


admin.site.register(Experiment, ExperimentAdmin)
admin.site.register(Session, SessionAdmin)
admin.site.register(Round, RoundAdmin)
admin.site.register(Vote, VoteAdmin)
admin.site.register(Stimulus, StimulusAdmin)
admin.site.register(StimulusGroup, StimulusGroupAdmin)
admin.site.register(StimulusVoteGroup, StimulusVoteGroupAdmin)
admin.site.register(Subject, SubjectAdmin)
admin.site.register(Experimenter, ExperimenterAdmin)

nest_site/nest/admin.py
Netflix/e2nest
from django.apps import AppConfig
from django.contrib.admin.apps import AdminConfig


class NestConfig(AppConfig):
    name = 'nest'
    default_site = 'nest.sites.NestSite'


class NestAdminConfig(AdminConfig):
    default_site = 'nest.sites.NestAdminSite'

nest_site/nest/apps.py
Netflix/e2nest
import os

from typing import List, Optional

from nest.helpers import validate_xml
from nest_site.settings import map_media_url_to_local

NEST_ROOT = os.path.dirname(os.path.realpath(__file__))


class NestConfig(object):

    @classmethod
    def root_path(cls, *components):
        return os.path.join(NEST_ROOT, *components)

    @classmethod
    def tests_resource_path(cls, *components):
        return cls.root_path('tests', 'resource', *components)

    @classmethod
    def media_path(cls, *components):
        return cls.root_path('..', 'media', *components)

    @classmethod
    def tests_workdir_path(cls, *components):
        return cls.root_path('tests', 'workdir', *components)

    @classmethod
    def resource_path(cls, *components):
        return cls.root_path('resource', *components)


class DisplayConfig(object):

    @staticmethod
    def show(**kwargs):
        from matplotlib import pyplot as plt
        if 'write_to_dir' in kwargs:
            format = kwargs['format'] if 'format' in kwargs else 'png'
            filedir = kwargs['write_to_dir'] if kwargs['write_to_dir'] is not None else NestConfig.tests_workdir_path('output')
            os.makedirs(filedir, exist_ok=True)
            for fignum in plt.get_fignums():
                fig = plt.figure(fignum)
                fig.savefig(os.path.join(filedir, str(fignum) + '.' + format), format=format)
        else:
            plt.show()


class StimulusConfig(object):

    def __init__(self, config: dict, **more):
        self.config: dict = config
        assert isinstance(self.config, dict)
        self._assert(**more)

    def _assert(self, **more):
        assert 'contents' in self.config
        assert 'stimuli' in self.config
        assert 'stimulusvotegroups' in self.config
        assert 'stimulusgroups' in self.config

        assert isinstance(self.contents, list)
        for c in self.contents:
            assert isinstance(c, dict)
            assert 'content_id' in c
            assert 'name' in c
        cids = self.content_ids
        assert len(set(cids)) == len(cids), \
            f'content_ids must be unique, but got: {cids}'

        assert isinstance(self.stimuli, list)
        for s in self.stimuli:
            assert isinstance(s, dict)
            assert 'stimulus_id' in s
            assert 'path' in s
            assert 'type' in s
            assert 'content_id' in s
            assert s['content_id'] in self.content_ids
        sids = self.stimulus_ids
        assert len(set(sids)) == len(sids), \
            f'stimulus_ids must be unique, but got: {sids}'

        if 'skip_path_check' in more and more['skip_path_check'] is True:
            pass
        else:
            for s in self.stimuli:
                url_path: str = s['path']
                if url_path.startswith('http://') or url_path.startswith('https://'):
                    # don't check publically hosted urls
                    continue
                local_path = map_media_url_to_local(url_path)
                assert os.path.exists(local_path), \
                    f"url path {url_path} should map to local path {local_path}, which does not exist"

        assert isinstance(self.stimulusvotegroups, list)
        for svg in self.stimulusvotegroups:
            assert isinstance(svg, dict)
            assert 'stimulusvotegroup_id' in svg
            assert 'stimulus_ids' in svg
            assert isinstance(svg['stimulus_ids'], list)
            for sid in svg['stimulus_ids']:
                assert sid in self.stimulus_ids
            assert len(svg['stimulus_ids']) in [1, 2], \
                "for now, only support stimulus_ids list length 1 " \
                "(single-stimulus test) or 2 (double-stimulus test)"
        svgids = self.stimulusvotegroup_ids
        assert len(set(svgids)) == len(svgids), \
            f'stimulusvotegroup_ids must be unique, but got: {svgids}'

        assert isinstance(self.stimulusgroups, list)
        for sg in self.stimulusgroups:
            assert isinstance(sg, dict)
            assert 'stimulusgroup_id' in sg
            assert 'stimulusvotegroup_ids' in sg
            assert isinstance(sg['stimulusvotegroup_ids'], list)
            for svgid in sg['stimulusvotegroup_ids']:
                assert svgid in self.stimulusvotegroup_ids
            if 'video_display_percentage' in sg:
                assert 0 < sg['video_display_percentage'] <= 100
        sgids = self.stimulusgroup_ids
        assert len(set(sgids)) == len(sgids), \
            f'stimulusgroup_ids must be unique, but got: {sgids}'

    @property
    def contents(self) -> List[dict]:
        return self.config['contents']

    @property
    def stimuli(self) -> List[dict]:
        return self.config['stimuli']

    @property
    def stimulusvotegroups(self) -> List[dict]:
        return self.config['stimulusvotegroups']

    @property
    def stimulusgroups(self) -> List[dict]:
        return self.config['stimulusgroups']

    @property
    def content_ids(self) -> List[int]:
        return [c['content_id'] for c in self.contents]

    @property
    def stimulus_ids(self) -> List[int]:
        return [s['stimulus_id'] for s in self.stimuli]

    @property
    def stimulusvotegroup_ids(self) -> List[int]:
        return [svg['stimulusvotegroup_id'] for svg in self.stimulusvotegroups]

    @property
    def stimulusgroup_ids(self) -> List[int]:
        return [sg['stimulusgroup_id'] for sg in self.stimulusgroups]

    def get_video_display_percentage(self, stimulusgroup_id: int) -> Optional[int]:
        sg: dict
        for sg in self.stimulusgroups:
            if sg['stimulusgroup_id'] == stimulusgroup_id:
                if 'video_display_percentage' in sg:
                    return sg['video_display_percentage']
                else:
                    return 100  # 100%, or full browser canvas
        else:
            return None


class ExperimentConfig(object):

    def __init__(self,
                 stimulus_config: StimulusConfig,
                 config: dict):
        self.stimulus_config: StimulusConfig = stimulus_config
        self.config: dict = config
        assert isinstance(self.config, dict)
        self._assert()

    def _assert(self):
        assert isinstance(self.title, str)

        if self.description is not None:
            assert isinstance(self.description, str)

        assert isinstance(self.rounds_per_session, int)
        assert self.rounds_per_session >= 1

        assert self.random_seed is None or isinstance(self.random_seed, int)

        from .models import Vote
        Vote.find_subclass(self.vote_scale)

        assert self.methodology in ['acr', 'acr5c', 'dcr', 'tafc', 'ccr', 'samviq', 'samviq5d']
        if self.methodology in ['acr', 'acr5c']:
            for svg in self.stimulus_config.stimulusvotegroups:
                assert len(svg['stimulus_ids']) == 1
        elif self.methodology in ['dcr', 'tafc', 'ccr']:
            for svg in self.stimulus_config.stimulusvotegroups:
                assert len(svg['stimulus_ids']) == 2
        elif self.methodology in ['samviq', 'samviq5d']:
            for svg in self.stimulus_config.stimulusvotegroups:
                assert len(svg['stimulus_ids']) == 2
            # must make sure that for each sg, all svgs share the same reference
            svgs = self.stimulus_config.stimulusvotegroups
            svg_dict = dict(zip(  # dict: svg_id -> svg
                [svg['stimulusvotegroup_id'] for svg in svgs], svgs))
            for sg in self.stimulus_config.stimulusgroups:
                second_sids = [svg_dict[svg_id]['stimulus_ids'][1]
                               for svg_id in sg['stimulusvotegroup_ids']]
                assert len(set(second_sids)) == 1, \
                    f"all svgs belong to one sg must share the same second " \
                    f"element in svg['stimulus_ids'], which is the reference:" \
                    f" {second_sids}"
        else:
            assert False

        try:
            if self.methodology in ['acr', 'dcr']:
                assert self.vote_scale in [
                    'THREE_POINT',
                    'FIVE_POINT',
                    'SEVEN_POINT',
                    'ELEVEN_POINT',
                ]
            elif self.methodology == 'tafc':
                assert self.vote_scale in ['2AFC']
            elif self.methodology == 'ccr':
                assert self.vote_scale in ['CCR_THREE_POINT']
            elif self.methodology == 'acr':
                assert self.vote_scale in ['FIVE_POINT']
            elif self.methodology == 'acr5c':
                assert self.vote_scale in ['0_TO_100']
            elif self.methodology == 'samviq':
                assert self.vote_scale in ['0_TO_100']
            elif self.methodology == 'samviq5d':
                assert self.vote_scale in ['FIVE_POINT']
            else:
                assert False
        except AssertionError:
            raise AssertionError(f'Unsupported (methodology, vote_scale): ({self.methodology}, {self.vote_scale})')

        self._assert_blocklist_stimulusgroup_ids()
        self._assert_training_round_ids()
        self._assert_prioritized()
        self._assert_additions()
        self._assert_round_context()
        self._assert_done_context()

    @property
    def title(self):
        assert 'title' in self.config
        return self.config['title']

    @property
    def description(self):
        return self.config['description'] \
            if 'description' in self.config else None

    @property
    def rounds_per_session(self):
        assert 'rounds_per_session' in self.config
        return self.config['rounds_per_session']

    @property
    def random_seed(self):
        return self.config['random_seed'] \
            if 'random_seed' in self.config else None

    @property
    def vote_scale(self):
        assert 'vote_scale' in self.config
        return self.config['vote_scale']

    @property
    def methodology(self):
        assert 'methodology' in self.config
        return self.config['methodology']

    @property
    def prioritized(self):
        """
        prioritized is a list of dictionaries in the format of
        [
            {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 1, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 1, 'round_id': None, 'stimulusgroup_id': 6},
        ],
        where 'session_idx': None represents that the rule applies to all sessions,
        and 'round_id': None represents that the rule applies to a randomized
        round within that session. The prioritized list dictates in which
        (session, round) to set the stimulusgroup to be voted, not subject to
        the ordering procedure by ExperimentOrder. Note that the later a
        dictionary appears in the list, the higher priority it has to override
        earlier dictionaries.

        Note that stimulusgroup_idx is different from stimulusgroup_id. The
        former has to take a value from [0, len(stimulusgroups) - 1], and the
        latter could take value from any non-negative integers.
        """
        return self.config['prioritized'] \
            if 'prioritized' in self.config else list()

    @property
    def blocklist_stimulusgroup_ids(self):
        """
        stimulusgroup_ids that appear in blocklist_stimulusgroup_ids are not
        selected unless they appear in the prioritized list.

        For example:
        to set sg 0 as training:
        1) add it to the prioritized list
        2) mark it in the blocklist_stimulusgroup_ids, such that it won't
        appear elsewhere
        """
        return self.config['blocklist_stimulusgroup_ids'] \
            if 'blocklist_stimulusgroup_ids' in self.config else list()

    @property
    def training_round_ids(self):
        """
        training_round_ids are the round_ids for rounds that are considered as
        "training" rounds. Usually this affects the messages displayed in the
        round titles. For example, if a round is a "training" round, the title
        could be "training 1 of 2"; if a round is a regular round, the title
        could be "round 1 of 8" (assume in rounds_per_session is 10).
        """
        return self.config['training_round_ids'] \
            if 'training_round_ids' in self.config else list()

    def _assert_training_round_ids(self):
        assert isinstance(self.training_round_ids, list)
        rounds_per_session = self.rounds_per_session
        for rid in self.training_round_ids:
            assert 0 <= rid < rounds_per_session

    def _assert_blocklist_stimulusgroup_ids(self):
        assert isinstance(self.blocklist_stimulusgroup_ids, list)
        stimulusgroup_ids = self.stimulus_config.stimulusgroup_ids
        for bsgid in self.blocklist_stimulusgroup_ids:
            assert isinstance(bsgid, int) and \
                bsgid in stimulusgroup_ids

    def _assert_prioritized(self):
        assert isinstance(self.prioritized, list)
        stimulusgroup_ids = self.stimulus_config.stimulusgroup_ids
        for d in self.prioritized:
            assert isinstance(d, dict)
            assert 'session_idx' in d
            assert 'round_id' in d
            assert 'stimulusgroup_id' in d
            session_idx = d['session_idx']
            round_id = d['round_id']
            stimulusgroup_id = d['stimulusgroup_id']
            assert session_idx is None or \
                   (isinstance(session_idx, int) and session_idx >= 0)
            assert round_id is None or \
                   (isinstance(round_id, int) and
                    0 <= round_id < self.rounds_per_session)
            assert isinstance(stimulusgroup_id, int) and \
                   stimulusgroup_id in stimulusgroup_ids

    @property
    def additions(self):
        """
        additions specify the steps in a test session that are not "rounds".
        This includes instruction pages, pre-/post-test surveys etc.
        """
        return self.config['additions'] \
            if 'additions' in self.config else list()

    @property
    def round_context(self):
        """
        round_context specifies the appearances (the page displayed) in each
        voting round.
        """
        return self.config['round_context'] \
            if 'round_context' in self.config else dict()

    @property
    def done_context(self):
        """
        done_conext specifies the appearances (the page display) when a session
        is done.
        """
        return self.config['done_context'] \
            if 'done_context' in self.config else dict()

    def _assert_additions(self):
        assert isinstance(self.additions, list)
        for addition in self.additions:
            assert 'position' in addition
            assert 'round_id' in addition['position']
            assert isinstance(addition['position']['round_id'], int)
            assert 0 <= addition['position']['round_id'] < self.rounds_per_session
            assert 'before_or_after' in addition['position']
            assert addition['position']['before_or_after'] in ['before', 'after']
            assert 'context' in addition
            assert 'title' in addition['context']
            assert 'text_html' in addition['context']
            assert 'actions_html' in addition['context']
            assert '{action_url}' in addition['context']['actions_html'], \
                "expect '{action_url}' in actions_html as template to be" \
                "filled during page rendering"

    def _assert_round_context(self):
        assert isinstance(self.round_context, dict)
        # leave other assertions to the Page initialization stage

    def _assert_done_context(self):
        assert isinstance(self.done_context, dict)
        if 'text_html' in self.done_context:
            assert validate_xml(self.done_context['text_html']), \
                f"invalid html: {self.done_context['text_html']}"

nest_site/nest/config.py
Netflix/e2nest
import random
from enum import Enum
from typing import List, Optional

import pandas
from django.db import transaction
from nest.config import ExperimentConfig
from nest.helpers import memoized, my_argmin
from nest.models import Content, Experiment, Round, Session, Stimulus, \
    StimulusGroup, StimulusVoteGroup, Subject, Vote


class SessionStatus(Enum):
    """
    A status of a Session could be one of the following:
    - UNINITIALIZED: Session is created but no Rounds are created and
    assigned a StimulusGroup.
    - PARTIALLY_INITIALIZED: Session is created and at least one (but not
    all) Rounds are created and assigned a StimulusGroup.
    - INITIALIZED: All Rounds are created and assigned a StimulusGroup.
    - PARTIALLY_FINISHED: Some (not all) Votes have been recorded for the
    Session.
    - FINISHED: All Votes have been recorded for the Session. This means that
    all StimulusVoteGroup associated withe the StimulusGroup of each Round
    has a Vote (foreign-keyed by that Round).

    Refer to the Data Model diagram for the relations between Round,
    StimulusGroup, StimulusVoteGroup and Vote:
    https://docs.google.com/presentation/d/1dvdsim38_p4zB_D1sIm3p8_0F9og7_VaKkDm-8g6Eps/edit#slide=id.ge6986ddd86_1_79
    """
    UNINITIALIZED = 1
    PARTIALLY_INITIALIZED = 2
    INITIALIZED = 3
    PARTIALLY_FINISHED = 4
    FINISHED = 5


class ExperimentController(object):
    """Controller of the business logic of an Experiment."""

    def __init__(self, experiment: Experiment,
                 experiment_config: ExperimentConfig,
                 ) -> None:
        self.experiment: Experiment = experiment
        self.experiment_config: ExperimentConfig = experiment_config
        self.randgen = random.Random(self.experiment_config.random_seed)

    def get_and_assert_current_stimulusgroups(self):
        # tools developed for testing

        # scan sg added by far, put them into dictionary of sgid as key, and
        # assert that the sgid is unique
        d_sgid_to_sg = dict()
        for s in self.experiment.session_set.all():
            for r in s.round_set.all():
                if r.stimulusgroup is not None and r.stimulusgroup.stimulusgroup_id is not None:
                    if r.stimulusgroup.stimulusgroup_id in d_sgid_to_sg:
                        assert d_sgid_to_sg[r.stimulusgroup.stimulusgroup_id] == r.stimulusgroup, \
                            'expect unique stimulusgroup with sgid {} but have at least two: {}, {}'.\
                            format(r.stimulusgroup.stimulusgroup_id,
                                   d_sgid_to_sg[r.stimulusgroup.stimulusgroup_id],
                                   r.stimulusgroup)
                    else:
                        d_sgid_to_sg[r.stimulusgroup.stimulusgroup_id] = \
                            r.stimulusgroup

        stimulusgroups = list()
        for stimulusgroup in self.experiment_config.stimulus_config.stimulusgroups:
            stimulusgroup_id = stimulusgroup['stimulusgroup_id']
            if stimulusgroup_id in d_sgid_to_sg:
                sg = d_sgid_to_sg[stimulusgroup_id]
                stimulusgroups.append(sg)
        return stimulusgroups

    def populate_stimuli(self):
        """
        populate Content, Stimulus, StimulusVoteGroup and StimulusGroup based on config.
        """

        cd: dict
        for cd in self.experiment_config.stimulus_config.contents:
            try:
                Content.objects.get(
                    experiment=self.experiment,
                    content_id=cd['content_id'])
            except Content.DoesNotExist:
                Content.objects.create(
                    experiment=self.experiment,
                    content_id=cd['content_id'])

        sd: dict
        for sd in self.experiment_config.stimulus_config.stimuli:
            try:
                Stimulus.objects.get(
                    experiment=self.experiment,
                    stimulus_id=sd['stimulus_id'])
            except Stimulus.DoesNotExist:
                c: Content = Content.objects.get(
                    experiment=self.experiment,
                    content_id=sd['content_id'])
                Stimulus.objects.create(
                    experiment=self.experiment,
                    stimulus_id=sd['stimulus_id'],
                    content=c,
                )  # TODO: add condition in future

        svgd: dict
        for svgd in self.experiment_config.stimulus_config.stimulusvotegroups:
            assert len(svgd['stimulus_ids']) in [1, 2], \
                'for now, can only deal with stimulus_ids of length 1 or 2, ' \
                'but got: {}'.format(len(svgd['stimulus_ids']))
            try:
                StimulusVoteGroup.objects.get(
                    experiment=self.experiment,
                    stimulusvotegroup_id=svgd['stimulusvotegroup_id'])
            except StimulusVoteGroup.DoesNotExist:
                if len(svgd['stimulus_ids']) == 1:
                    s: Stimulus = Stimulus.objects.get(
                        experiment=self.experiment,
                        stimulus_id=svgd['stimulus_ids'][0])
                    StimulusVoteGroup.create_stimulusvotegroup_from_stimulus(
                        s,
                        stimulusvotegroup_id=svgd['stimulusvotegroup_id'],
                        experiment=self.experiment)
                elif len(svgd['stimulus_ids']) == 2:
                    s1: Stimulus = Stimulus.objects.get(
                        experiment=self.experiment,
                        stimulus_id=svgd['stimulus_ids'][0])
                    s2: Stimulus = Stimulus.objects.get(
                        experiment=self.experiment,
                        stimulus_id=svgd['stimulus_ids'][1])
                    StimulusVoteGroup.create_stimulusvotegroup_from_stimuli_pair(
                        s1, s2,
                        stimulusvotegroup_id=svgd['stimulusvotegroup_id'],
                        experiment=self.experiment)
                else:
                    assert False

        sgd: dict
        for sgd in self.experiment_config.stimulus_config.stimulusgroups:
            assert len(sgd['stimulusvotegroup_ids']) > 0
            try:
                StimulusGroup.objects.get(
                    experiment=self.experiment,
                    stimulusgroup_id=sgd['stimulusgroup_id'])
            except StimulusGroup.DoesNotExist:
                sg = StimulusGroup.objects.create(
                    experiment=self.experiment,
                    stimulusgroup_id=sgd['stimulusgroup_id'])
                for svgid in sgd['stimulusvotegroup_ids']:
                    svg = StimulusVoteGroup.objects.get(
                        experiment=self.experiment,
                        stimulusvotegroup_id=svgid)
                    assert svg.stimulusgroup is None, \
                        'svg.stimulusgroup should not be overwritten'
                    svg.stimulusgroup = sg
                    svg.save()

    @transaction.atomic
    def add_session(self, subject: Subject):
        """
        add a new Session to Experiment, and assign to subject. A new Session
        is created, so are the corresponding Rounds.
        """
        ordering_so_far = self._get_ordering_so_far()
        stimulusgroup_ids = [sg['stimulusgroup_id'] for sg in
                             self.experiment_config.stimulus_config.stimulusgroups]
        d_rid_to_sgid = self._order(
            rounds_per_session=self.experiment_config.rounds_per_session,
            stimulusgroup_ids=stimulusgroup_ids,
            subject_id=subject.id,
            ordering_so_far=ordering_so_far,
            prioritized=self.experiment_config.prioritized,
            random_seed=self.randgen.randint(0, 2**16),
            blocklist_stimulusgroup_ids=self.experiment_config.blocklist_stimulusgroup_ids,
        )

        sess = Session(experiment=self.experiment, subject=subject)
        sess.save()
        for rid, sgid in d_rid_to_sgid.items():
            sg: StimulusGroup = StimulusGroup.objects.get(
                stimulusgroup_id=sgid, experiment=sess.experiment)
            r = Round(session=sess,
                      round_id=rid,
                      stimulusgroup=sg)
            r.save()
        return sess

    @transaction.atomic
    def delete_session(self, session_id):
        """
        Delete a Session, and the corresponding Rounds and Votes if exist.
        """
        sess: Session = Session.objects.get(id=session_id)
        sess.delete()

    def _get_ordering_so_far(self):
        """
        retrieve ordering so far from db, in the following format:
        [
            {'subject': subject_id, 'stimulusgroups': dict_of_round_id_to_stimulusgroup_id}
            ...
        ]
        """
        ordering = []
        s: Session
        for s in self.experiment.session_set.all():
            sess_id = s.id
            od = self._get_ordering_for_session(sess_id)
            ordering.append(od)
        return ordering

    @memoized
    def _get_ordering_for_session(self, sess_id):
        s = Session.objects.get(id=sess_id)
        subject_id = s.subject.id
        sg_dict = dict()
        r: Round
        for r in s.round_set.all():
            sg = r.stimulusgroup
            sg_dict[r.round_id] = sg.stimulusgroup_id
        for rid in range(self.experiment_config.rounds_per_session):
            assert rid in sg_dict, \
                'round ids must be 0, ..., {}, but are: {}'.format(
                    self.experiment_config.rounds_per_session - 1,
                    sg_dict.keys())
        od = {
            'subject': subject_id,
            'stimulusgroups': sg_dict,
        }
        return od

    @staticmethod
    def _order(
            rounds_per_session: int,
            stimulusgroup_ids: List[int],
            subject_id: int,
            ordering_so_far: List[dict],
            prioritized: List[dict],
            random_seed: Optional[int],
            blocklist_stimulusgroup_ids: List[int],
    ):
        """
        return new stimulusgroup assignment in the format of:
        dict: round_id -> stimulusgroup_id
        """

        SUBJECT_WEIGHT = 5

        num_sessions_sofar = len(ordering_so_far)
        new_session_id = num_sessions_sofar

        # Note that stimulusgroup_idx is different from stimulusgroup_id. The
        # former has to take a value from [0, len(stimulusgroups) - 1], and the
        # latter could take value from any non-negative integers.
        d_sgid_to_sgidx = dict(zip(stimulusgroup_ids, range(len(stimulusgroup_ids))))
        d_sgidx_to_sgid = dict(zip(range(len(stimulusgroup_ids)), stimulusgroup_ids))

        # construct weight histogram:
        # use the heuristic rule to prioritize sg to test based on 5 * x + y,
        # where x is the current subject's count, and y is the overall count
        wt_hist = [0 for _ in stimulusgroup_ids]
        for pd in ordering_so_far:
            for sgid in pd['stimulusgroups'].values():
                wt_hist[d_sgid_to_sgidx[sgid]] += 1
                if subject_id == pd['subject']:
                    wt_hist[d_sgid_to_sgidx[sgid]] += SUBJECT_WEIGHT

        # output:
        d_rid_to_sgid = dict()  # output: round determined
        l_sgid = list()  # output: round not determined yet

        # fill in positions specified in prioritized list:
        for pd in prioritized:
            if (pd['session_idx'] is None  # every session needs to be applied
                    or pd['session_idx'] == new_session_id):
                sgid = pd['stimulusgroup_id']
                rid = pd['round_id']
                if rid is None:
                    l_sgid.append(sgid)
                else:
                    d_rid_to_sgid[rid] = sgid
        for rid, sgid in d_rid_to_sgid.items():
            # since it is the same subject, the weight increment is the
            # 5 * x + y, where x = 1 and y = 1
            wt_hist[d_sgid_to_sgidx[sgid]] += SUBJECT_WEIGHT + 1
        for sgid in l_sgid:
            wt_hist[d_sgid_to_sgidx[sgid]] += SUBJECT_WEIGHT + 1

        random.seed(random_seed)

        remaining_rounds_to_fulfill = rounds_per_session - len(d_rid_to_sgid) - len(l_sgid)

        sgidx_candidates = list()
        for _ in range(remaining_rounds_to_fulfill):

            # replenish if current sg candidates are empty
            if len(sgidx_candidates) == 0:
                new_sgidxs = list(range(len(stimulusgroup_ids)))
                if blocklist_stimulusgroup_ids is not None:
                    for bsgid in blocklist_stimulusgroup_ids:
                        new_sgidxs.remove(d_sgid_to_sgidx[bsgid])
                sgidx_candidates += new_sgidxs

            # find from sg candidates that has least weight
            sgidxs = my_argmin(wt_hist, sgidx_candidates)
            assert len(sgidxs) > 0
            if len(sgidxs) == 1:
                sgidx = sgidxs[0]
            else:
                sgidx = sgidxs[random.randint(0, len(sgidxs) - 1)]
            l_sgid.append(d_sgidx_to_sgid[sgidx])
            # since it is the same subject, the weight increment is the
            # 5 * x + y, where x = 1 and y = 1
            wt_hist[sgidx] += SUBJECT_WEIGHT + 1
            sgidx_candidates.remove(sgidx)

        # lastly, randomly assign sgidx in l to different rid
        random.shuffle(l_sgid)
        for rid in range(rounds_per_session):
            if rid in d_rid_to_sgid:
                continue
            d_rid_to_sgid[rid] = l_sgid[0]
            del l_sgid[0]
        assert len(l_sgid) == 0

        return d_rid_to_sgid

    def get_session_status(self, session: Session) -> SessionStatus:
        rounds_per_session = self.experiment_config.rounds_per_session
        rounds_existed = [False for _ in range(rounds_per_session)]
        r: Round
        rounds: List[Round] = list(session.round_set.all())
        for r in rounds:
            assert 0 <= r.round_id < rounds_per_session
            rounds_existed[r.round_id] = True
        if all(rounds_existed):
            pass
        elif all([not e for e in rounds_existed]):
            return SessionStatus.UNINITIALIZED
        else:
            return SessionStatus.PARTIALLY_INITIALIZED

        # iterate through all StimulusVoteGroups. If all assgined Votes, return
        # 'done'; if none, return 'initialized'; otherwise, return
        # 'done_partially'.
        all_true = None
        all_false = None
        for r in rounds:
            svg: StimulusVoteGroup
            for svg in r.stimulusgroup.stimulusvotegroup_set.all():
                votes = Vote.objects.filter(round=r, stimulusvotegroup=svg)
                if votes.count() == 0:
                    all_true = False
                elif votes.count() == 1:
                    all_false = False
                else:
                    assert False, \
                        "must have at most one Vote per StimulusVoteGroup" \
                        "but got {}: {}, {}".format(len(votes.count()),
                                                    svg, votes.all())
                if all_true is False and all_false is False:
                    return SessionStatus.PARTIALLY_FINISHED

        if all_true is False and all_false is None:
            return SessionStatus.INITIALIZED
        elif all_true is None and all_false is False:
            return SessionStatus.FINISHED
        elif all_true is None and all_false is None:
            # this happens when none of svg has been iterated
            return SessionStatus.INITIALIZED
        else:
            assert False

    @staticmethod
    def reset_session(session: Session):
        """reset session by deleting votes"""
        r: Round
        v: Vote
        for r in session.round_set.all():
            for v in r.vote_set.all():
                v.delete()
        return session

    def get_session_steps(self, session: Session) -> list:
        """
        return a list of steps for the session, including both regular rounds
        and additions (instruction steps and pre-/post-test surveys).
        """
        od = self._get_ordering_for_session(session.id)
        assert 'stimulusgroups' in od
        assert isinstance(od['stimulusgroups'], dict)
        steps = list()
        for rid in sorted(od['stimulusgroups'].keys()):
            sgid = od['stimulusgroups'][rid]
            steps.append({
                'position': {
                    'round_id': rid,
                },
                'context': {
                    'stimulusgroup_id': sgid,
                }
            })
        additions = self.experiment_config.additions

        for i in range(len(steps) - 1):
            assert steps[i]['position']['round_id'] == \
                   steps[i + 1]['position']['round_id'] - 1

        for addition in additions:
            self._insert_addition_into_steps(addition, steps)

        return steps

    @staticmethod
    def _insert_addition_into_steps(addition, steps):
        round_id = addition['position']['round_id']
        before_or_after = addition['position']['before_or_after']
        # advance pos in steps, insert addition in the proper location
        if before_or_after == 'before':
            # Rule: find the first element with the same 'round_id' but
            # not with 'before_or_after' == 'before', insert before that
            # element.
            #
            # Examples:
            # (b0 means "element already inserted before round_id 0,
            #  a1 means "element already inserted after round_id 1)
            # b0, b0, 0, 1, 2, 3, 4
            #        ^
            #        |
            #   {'round_id': 0, 'before_or_after': 'before'}
            #
            # b0, b0, 0, a0, a0, b1, 1, a1, 2, 3, 4
            #                       ^
            #                       |
            #             {'round_id': 1, 'before_or_after': 'before'}
            #
            # raise exception if:
            # b0, b0, 0, a0, a0, b1, 1, a1, 2, 3, 4
            #                                       ^
            #                                       |
            #             {'round_id': 5, 'before_or_after': 'before'}
            #
            # raise exception if:
            #  b0, b0, 0, 1, 2, 3, 4
            # ^
            # |
            # {'round_id': -1, 'before_or_after': 'before'}

            for pos in range(len(steps)):
                if steps[pos]['position']['round_id'] < round_id or \
                        (round_id == steps[pos]['position']['round_id']
                         and 'before_or_after' in steps[pos]['position']
                         and steps[pos]['position']['before_or_after'] == 'before'):
                    continue
                else:
                    assert round_id == steps[pos]['position']['round_id']
                    steps.insert(pos, addition)
                    break
            else:
                # pos is at the tail of array
                assert False  # never expect insert at the end

        elif before_or_after == 'after':
            # Rule: find the first element with round_id + 1, insert before
            # that element; if reaching the end of the list, ensure that
            # the last element of the list has the same round_id (raise
            # exception if not), and insert at the end of the list.
            #
            # Examples:
            # b0, b0, 0, a0, 1, 2, 3, 4
            #               ^
            #               |
            #   {'round_id': 0, 'before_or_after': 'after'}
            #
            # b0, b0, 0, a0, a0, b1, 1, a1, 2, 3, 4, a4
            #                                          ^
            #                                          |
            #             {'round_id': 4, 'before_or_after': 'after'}
            #
            # raise exception if:
            # b0, b0, 0, a0, a0, b1, 1, a1, 2, 3, 4
            #                                       ^
            #                                       |
            #             {'round_id': 5, 'before_or_after': 'after'}
            #
            # raise exception if:
            # b0, b0, 0, a0, a0, b1, 1, a1, 2, 3, 4
            # ^
            # |
            # {'round_id': -1, 'before_or_after': 'after'}
            for pos in range(len(steps)):
                if steps[pos]['position']['round_id'] < round_id + 1:
                    continue
                else:
                    assert pos != 0  # never expect insert at the beginning
                    assert steps[pos]['position']['round_id'] == round_id + 1
                    steps.insert(pos, addition)
                    break
            else:
                # pos is at tail of array
                assert steps[-1]['position']['round_id'] == round_id
                steps.insert(len(steps), addition)
        else:
            assert False

    @staticmethod
    def get_session_info(s: Session) -> dict:

        info = dict()

        info['session_id'] = s.id
        info['subject'] = s.subject.get_subject_name()

        rounds = info.setdefault('rounds', [])
        r: Round
        for r in s.round_set.all():
            rd = dict()
            rd['round_id'] = r.round_id
            rd['stimulusgroup_id'] = r.stimulusgroup.stimulusgroup_id

            svgs = rd.setdefault('stimulusvotegroups', [])
            svg: StimulusVoteGroup
            for svg in r.stimulusgroup.stimulusvotegroup_set.all():
                svgd = dict()
                svgd['stimulusvotegroup_id'] = svg.stimulusvotegroup_id
                try:
                    vote = Vote.objects.get(round=r, stimulusvotegroup=svg)
                    svgd['vote'] = vote.score
                except Vote.DoesNotExist:
                    pass
                svgs.append(svgd)

            rounds.append(rd)

        return info

    def get_stimuli_info(self):
        return {
            'contents': self.experiment_config.stimulus_config.contents,
            'stimuli': self.experiment_config.stimulus_config.stimuli,
            'stimulusvotegroups': self.experiment_config.stimulus_config.stimulusvotegroups,
            'stimulusgroups': self.experiment_config.stimulus_config.stimulusgroups,
        }

    def get_experiment_info(self) -> dict:
        """
        Get experiment information. This is the most compact but not quite
        readable way of presenting info.
        """
        info = dict()

        info['title'] = self.experiment.title
        info['description'] = self.experiment.description

        # TODO: add experimenter

        sessions = info.setdefault('sessions', [])
        s: Session
        for s in self.experiment.session_set.all():
            sinfo = self.get_session_info(s)
            sessions.append(sinfo)

        info['stimuli_info'] = self.get_stimuli_info()

        return info

    @staticmethod
    def denormalize_experiment_info(experiment_info: dict) -> pandas.DataFrame:
        """
        flattern the experiment_info created from get_experiment_info() to make
        human readable pandas DataFrame style.
        """
        stimuli_info = experiment_info['stimuli_info']
        assert 'contents' in stimuli_info
        assert 'stimuli' in stimuli_info
        assert 'stimulusvotegroups' in stimuli_info

        # build dict of dict:
        dd_s = dict(zip([s['stimulus_id'] for s in stimuli_info['stimuli']], stimuli_info['stimuli']))
        dd_svg = dict(zip([s['stimulusvotegroup_id'] for s in stimuli_info['stimulusvotegroups']], stimuli_info['stimulusvotegroups']))

        sessions: list = experiment_info['sessions']

        rows = list()
        for sess_info in sessions:
            assert 'subject' in sess_info
            assert 'rounds' in sess_info
            rd: dict
            for rd in sess_info['rounds']:
                assert 'round_id' in rd
                assert 'stimulusvotegroups' in rd
                assert 'stimulusgroup_id' in rd
                svgds: list = rd['stimulusvotegroups']
                svgd: dict
                for svgd in svgds:
                    assert 'stimulusvotegroup_id' in svgd
                    d_svg: dict = dd_svg[svgd['stimulusvotegroup_id']]
                    assert 'stimulus_ids' in d_svg
                    sids = d_svg['stimulus_ids']
                    assert len(sids) in [1, 2], f'for now, only export case of one or two stimuli per stimulusvotegroup, but {sids}'
                    if len(sids) == 1:
                        sid = sids[0]
                        path = dd_s[sid]['path']
                        sid2 = None
                        path2 = None
                    elif len(sids) == 2:
                        sid = sids[0]
                        sid2 = sids[1]
                        path = dd_s[sid]['path']
                        path2 = dd_s[sid2]['path']
                    else:
                        assert False
                    row = {
                        'subject': sess_info['subject'],
                        'session_id': sess_info['session_id'],
                        'round_id': rd['round_id'],
                        'sg_id': rd['stimulusgroup_id'],
                        'svg_id': svgd['stimulusvotegroup_id'],
                        'vote': svgd['vote'] if 'vote' in svgd else None,
                        'sid': sid,
                        'path': path,
                    }
                    if sid2 is not None:
                        row['sid2'] = sid2
                        row['path2'] = path2
                    rows.append(row)
        return pandas.DataFrame(rows)

nest_site/nest/control.py
Netflix/e2nest
import os
import re

from abc import ABCMeta, abstractmethod
from collections.abc import Hashable
from functools import partial
from io import StringIO
from typing import List, Optional

import matplotlib.pyplot as plt
from lxml import etree


# ==== decorators ====

def override(interface_class):
    def overrider(method):
        assert (method.__name__ in dir(interface_class)), \
            f"{method.__name__} does not override any method " \
            f"in {interface_class.__name__}"
        return method
    return overrider


# ==== mixins ====


class TypeVersionEnabled(object):
    """
    Mandate a type name and a version string. Derived class (e.g. an Executor)
    has a unique string combining type and version. The string is useful in
    identifying a Result by which Executor it is generated (e.g. VMAF_V0.1,
    PSNR_V1.0, or VMAF_feature_V0.1).
    """

    __metaclass__ = ABCMeta

    @property
    @abstractmethod
    def TYPE(self):
        raise NotImplementedError

    @property
    @abstractmethod
    def VERSION(self):
        raise NotImplementedError

    def __init__(self):
        self._assert_type_version()

    def _assert_type_version(self):

        assert re.match(r"^[a-zA-Z0-9_]+$", self.TYPE), \
            "TYPE can only contains alphabets, numbers and underscore (_)."

        assert re.match(r"^[a-zA-Z0-9._-]+$", self.VERSION), \
            "VERSION can only contains alphabets, numbers, dot (.), " \
            "hyphen(-) and underscore (_)."

    def get_type_version_string(self):
        return "{type}_V{version}".format(type=self.TYPE,
                                          version=self.VERSION)

    def get_cozy_type_version_string(self):
        return "{type} VERSION {version}".format(type=self.TYPE,
                                                 version=self.VERSION)

    @classmethod
    def find_subclass(cls, subclass_type):
        matched_subclasses = []
        for subclass in cls.get_subclasses_recursively():
            if hasattr(subclass, 'TYPE') and subclass.TYPE == subclass_type:
                matched_subclasses.append(subclass)
        assert len(matched_subclasses) == 1, \
            "Must have one and only one subclass of {class_name} with type " \
            "{type}, but got {num}".format(
                class_name=cls.__name__,
                type=subclass_type,
                num=len(matched_subclasses))
        return matched_subclasses[0]

    @classmethod
    def get_subclasses_recursively(cls):
        subclasses = cls.__subclasses__()
        subsubclasses = []
        for subclass in subclasses:
            subsubclasses += subclass.get_subclasses_recursively()
        return subclasses + subsubclasses


# ==== helpers ====


def get_file_name_without_extension(path):
    """

    >>> get_file_name_without_extension('yuv/src01_hrc01.yuv')
    'src01_hrc01'
    >>> get_file_name_without_extension('yuv/src01_hrc01')
    'src01_hrc01'
    >>> get_file_name_without_extension('abc/xyz/src01_hrc01.yuv')
    'src01_hrc01'
    >>> get_file_name_without_extension('abc/xyz/src01_hrc01.sdr.yuv')
    'src01_hrc01.sdr'
    >>> get_file_name_without_extension('abc/xyz/src01_hrc01.sdr.dvi.yuv')
    'src01_hrc01.sdr.dvi'

    """
    return os.path.splitext(path.split("/")[-1])[0]


def import_python_file(filepath):
    """
    Import a python file as a module.
    :param filepath:
    :return:
    """
    filename = get_file_name_without_extension(filepath)
    try:
        from importlib.machinery import SourceFileLoader
        ret = SourceFileLoader(filename, filepath).load_module()
    except ImportError:
        import imp
        ret = imp.load_source(filename, filepath)
    return ret


def empty_object():
    return type('', (), {})()


def my_argmin(a: List[int], shortlist: Optional[List[int]] = None):
    """
    My argmin implementation: return a list of indices chosen from the shortlist
    with the minimum value from a. If shortlist is None, use all indices.

    >>> my_argmin([1, 1, 2, 4])
    [0, 1]
    >>> my_argmin([3, 1, 2, 1])
    [1, 3]
    >>> my_argmin([1, 1, 2, 4], [0, 1, 3])
    [0, 1]
    >>> my_argmin([1, 1, 2, 4], [1, 2, 3])
    [1]
    >>> my_argmin([1, 1, 2, 4], [2, 3])
    [2]
    >>> my_argmin([1, 1, 2, 4], [0, 2, 3])
    [0]
    >>> my_argmin([1, 1, 2, 4], [])
    []
    """
    argmin = list()
    minval = None
    for i, e in enumerate(a):
        if shortlist is None or i in shortlist:
            if minval is None:
                minval = e
                argmin.append(i)
            else:
                if e == minval:
                    argmin.append(i)
                elif e < minval:
                    minval = e
                    argmin = [i]
                else:
                    pass
    return argmin


def indices(a, func):
    """
    Get indices of elements in an array which satisfies func
    >>> indices([1, 2, 3, 4], lambda x: x>2)
    [2, 3]
    >>> indices([1, 2, 3, 4], lambda x: x==2.5)
    []
    >>> indices([1, 2, 3, 4], lambda x: x>1 and x<=3)
    [1, 2]
    >>> indices([1, 2, 3, 4], lambda x: x in [2, 4])
    [1, 3]
    >>> indices([1,2,3,1,2,3,1,2,3], lambda x: x > 2)
    [2, 5, 8]
    """
    return [i for (i, val) in enumerate(a) if func(val)]


class memoized(object):
    """ Decorator. Caches a function's return value each time it is called.
    If called later with the same arguments, the cached value is returned
    (not reevaluated).

    memoized is similar to persist, but if applied to
    class methods, persist will cache on a per-class basis, while memoized
    will cache on a per-object basis.

    Taken from: https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize
    """

    def __init__(self, func):
        self.func = func
        self.cache = {}

    def __call__(self, *args):
        if not isinstance(args, Hashable):
            return self.func(*args)
        if args in self.cache:
            return self.cache[args]
        else:
            value = self.func(*args)
            self.cache[args] = value
            return value

    def __repr__(self):
        """ Return the function's docstring. """
        return self.func.__doc__

    def __get__(self, obj, objtype):
        """ Support instance methods. """
        return partial(self.__call__, obj)


def map_path_to_noise_rmse(path: str) -> Optional[int]:
    """
    >>> map_path_to_noise_rmse('snowpiercer-filmgrain-subj-test-prep-101a-beamr5-20211020aopalach-__beamr5__1920_1080__ni_denoise_filter_before_scale__qp18_vmaf80.84_psnr36.08_kbps13225.28_fps23.976023976023978.mp4')  # noqa E501
    0
    >>> map_path_to_noise_rmse('snowpiercer-filmgrain-subj-test-prep-101a-beamr5-20211020aopalach-__beamr5__1920_1080__none__qp18_vmaf97.58_psnr38.41_kbps21499.17_fps23.976023976023978.mp4')  # noqa E501
    -1
    >>> map_path_to_noise_rmse('snowpiercer-filmgrain-subj-test-prep-101c-beamr5-20211020aopalach-__beamr5__1920_1080__ni_denoise_with_target_noise_rmse_5_filter_before_scale__qp18_vmaf87.62_psnr37.06_kbps13956.47_fps23.976023976023978.mp4')  # noqa E501
    5
    >>> map_path_to_noise_rmse('snowpiercer-filmgrain-subj-test-prep-101c-beamr5-20211020aopalach-__beamr5__1920_1080__ni_denoise_with_target_noise_rmse_10_filter_before_scale__qp18_vmaf94.54_psnr38.15_kbps18356.45_fps23.976023976023978.mp4')  # noqa E501
    10
    >>> map_path_to_noise_rmse('CTS3E1_B__15_55_16_0__SdrX264MainCrfVE__384_216__qp20_vmaf42.63_psnr27.55_kbps308.96_fps23.976.mp4')
    -1
    """
    import re
    mo = re.search(r"ni_denoise_with_target_noise_rmse_([0-9]*)", path)
    if mo:
        rmse = int(mo.group(1))
        assert rmse >= 0
        return rmse
    mo = re.search(r"_none_", path)
    if mo:
        return -1
    mo = re.search(r"ni_denoise_filter_before_scale", path)
    if mo:
        return 0
    return -1


def map_filename_to_type(path: str) -> Optional[str]:
    """
    >>> map_filename_to_type('/media/mp4/test1a1b/training/reference/documentary-now-season-1-the-e-filmgrain-subj-test-prep-101a-beamr5-20211020aopalach-__beamr5__1920_1080__none__qp18_vmaf95.07_psnr38.65_kbps16175.49_fps23.976023976023978.mp4')  # noqa E501
    'training'
    >>> map_filename_to_type('/media/mp4/test1a1b/testing/reference/friends-season-2-the-one-with--filmgrain-subj-test-prep-101a-beamr5-20211020aopalach-__beamr5__1920_1080__none__qp18_vmaf95.85_psnr47.4_kbps10704.94_fps23.976023976023978.mp4')  # noqa E501
    'testing'
    >>> map_filename_to_type('/media/mp4/test1a1b/reliability/PeeWeeS2Store-underVMAF-A-filmgrain-subj-test-prep-26-beamr5-20211007-__beamr5__960_540__none__qp37_vmaf26.91_psnr27.9_kbps283.91_fps29.97002997002997.mp4')  # noqa E501
    'reliability'
    >>> map_filename_to_type('snowpiercer-filmgrain-subj-test-prep-101a-beamr5-20211020aopalach-__beamr5__1920_1080__ni_denoise_filter_before_scale__qp18_vmaf80.84_psnr36.08_kbps13225.28_fps23.976023976023978.mp4')  # noqa E501
    Traceback (most recent call last):
    ...
    AssertionError
    >>> map_filename_to_type('CTS3E1_B__15_55_16_0__SdrX264MainCrfVE__384_216__qp20_vmaf42.63_psnr27.55_kbps308.96_fps23.976.mp4')
    Traceback (most recent call last):
    ...
    AssertionError
    """
    if '/training/' in path:
        return 'training'
    elif '/testing/' in path:
        return 'testing'
    elif '/reliability/' in path:
        return 'reliability'
    else:
        assert False


def map_participantid_to_test(participantid: str) -> Optional[str]:
    """
    >>> map_participantid_to_test('Netflix_Noise_0')
    '1a'
    >>> map_participantid_to_test('Netflix_Noise_1')
    '1b'
    >>> map_participantid_to_test('Netflix_Noise_2')
    '1a'
    >>> map_participantid_to_test('Netflix_Noise_3')
    '1b'
    >>> map_participantid_to_test('xyz')
    Traceback (most recent call last):
    ...
    AssertionError
    """
    mo = re.match(r"^Netflix_Noise_([0-9]*)", participantid)
    assert mo is not None
    pid = int(mo.group(1))
    if pid % 2 == 0:
        return '1a'
    else:
        return '1b'


def sanity_same_content(enc_filepath: str, ref_filepath: str) -> bool:
    """
    >>> sanity_same_content("/Users/zli/Data/noise_study/projects_6sec_07262021/encode/selected_x264eaas_prefilter_0/oishinbo_season_2_episode_43__4_3_4_9__SdrX264MainCrfVE__1920_1080__qp22_enable_audio_False_vmaf90.64_phonevmaf99.79_psnr41.30_kbps2456.31.mp4",  # noqa E501
    ... "/Users/zli/Data/noise_study/projects_6sec_07262021/encode/selected_beamr5fixed/oishinbo_season_2_episode_43/oishinbo-season-2-episode-43-filmgrain-subj-test-prep-102c-beamr5-20211020aopalach-__beamr5__1920_1080__ni_denoise_with_target_noise_rmse_2_filter_before_scale__qp37_vmaf80.53_psnr39.58_kbps155.77_fps23.976023976023978.mp4")  # noqa E501
    True
    >>> sanity_same_content("/Users/zli/Data/noise_study/projects_6sec_07262021/encode/selected_x264eaas_prefilter_0/oishinbo_season_2_episode_43__4_3_4_9__SdrX264MainCrfVE__1920_1080__qp22_enable_audio_False_vmaf90.64_phonevmaf99.79_psnr41.30_kbps2456.31.mp4",  # noqa E501
    ... "/Users/zli/Data/noise_study/projects_6sec_07262021/encode/selected_beamr5fixed/groundhog_day/groundhog-day-filmgrain-subj-test-prep-102-4-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp23_vmaf84.8_psnr35.36_kbps11438.17_fps23.976023976023978.mp4")  # noqa E501
    False
    >>> sanity_same_content("/Users/zli/Data/noise_study/projects_6sec_07262021/encode/selected_beamr5fixed/the_walking_dead_season_6_the_/the-walking-dead-season-6-the--filmgrain-subj-test-prep-102a-beamr5-20211020aopalach-__beamr5__1920_1080__none__qp22_vmaf81.55_psnr36.88_kbps6048.37_fps23.976023976023978.mp4",  # noqa E501
    ... "/Users/zli/Data/noise_study/projects_6sec_07262021/encode/selected_beamr5fixed/the_guy_in_the_grave_next_door/the-guy-in-the-grave-next-door-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp28_vmaf86.0_psnr36.22_kbps7769.82_fps24.0.mp4")  # noqa E501
    False
    >>> sanity_same_content("/Users/zli/Data/noise_study/projects_6sec_07262021/other/encode_selection/test1a1b/testing/the-walking-dead-season-6-the--filmgrain-subj-test-prep-101c-beamr5-20211020aopalach-__beamr5__1920_1080__ni_denoise_with_target_noise_rmse_5_filter_before_scale__qp18_vmaf84.35_psnr38.63_kbps12859.57_fps23.976023976023978.mp4",  # noqa E501
    ... " /Users/zli/Data/noise_study/projects_6sec_07262021/other/encode_selection/test1a1b/testing/reference/the-walking-dead-season-6-the--filmgrain-subj-test-prep-101a-beamr5-20211020aopalach-__beamr5__1920_1080__none__qp18_vmaf89.07_psnr39.53_kbps19680.52_fps23.976023976023978.mp4")  # noqa E501
    True
    """
    enc_filename = os.path.splitext(os.path.basename(enc_filepath))[0]
    ref_filename = os.path.splitext(os.path.basename(ref_filepath))[0]
    enc_set = {_get_first_nontrivial_word(enc_filename, '-'),
               _get_first_nontrivial_word(enc_filename, '_')}
    ref_set = {_get_first_nontrivial_word(ref_filename, '-'),
               _get_first_nontrivial_word(ref_filename, '_')}
    return len(enc_set.intersection(ref_set).difference({'the', 'in'})) > 0


def _get_first_nontrivial_word(enc_filename, splitter='-'):
    words = enc_filename.split(splitter)
    words = list(filter(lambda w: w.lower() not in ['the', 'in', 'a'], words))
    assert len(words) > 0
    return words[0]


def map_filename_to_bitrate_kbps(path: str) -> Optional[float]:
    """
    >>> map_filename_to_bitrate_kbps('/media/mp4/joint_av_quality_v1/MasterOfNoneS1E2__frm20000to20252__SdrVp9Eve2pVE__608_342__100_videoChunksizeInSeconds_10_vmaf22.85_phonevmaf37.40_psnr28.98_kbps98.74.mp4')  # noqa E501
    98.74
    >>> map_filename_to_bitrate_kbps('/media/mp4/joint_av_quality_v1/reference/IpMan2__frm57060to57300__SdrVp9Eve2pVE__1920_1080__7500_audio_kbps_192_videoChunksizeInSeconds_10_vmaf99.51_phonevmaf100.00_psnr38.85_kbps7292.57.mp4')  # noqa E501
    7292.57
    >>> map_filename_to_bitrate_kbps('/media/mp4/joint_av_quality_v1/reference/IpMan2__frm57060to57300__SdrVp9Eve2pVE__1920_1080__7500_audio_kbps_192_videoChunksizeInSeconds_10_vmaf99.51_phonevmaf100.00_psnr38.85_mbps7292.57.mp4')  # noqa E501
    """
    basename = os.path.basename(path)
    basename_no_ext = os.path.splitext(basename)[0]
    mo = re.search(r"_kbps([0-9.]+)", basename_no_ext)
    if mo is None:
        return None
    bitrate_kbps = float(mo.group(1))
    return bitrate_kbps


def map_filename_to_vmaf(path: str) -> Optional[float]:
    """
    >>> map_filename_to_vmaf('/media/mp4/joint_av_quality_v1/MasterOfNoneS1E2__frm20000to20252__SdrVp9Eve2pVE__608_342__100_videoChunksizeInSeconds_10_vmaf22.85_phonevmaf37.40_psnr28.98_kbps98.74.mp4')  # noqa E501
    22.85
    >>> map_filename_to_vmaf('/media/mp4/joint_av_quality_v1/reference/IpMan2__frm57060to57300__SdrVp9Eve2pVE__1920_1080__7500_audio_kbps_192_videoChunksizeInSeconds_10_vmaf99.51_phonevmaf100.00_psnr38.85_kbps7292.57.mp4')  # noqa E501
    99.51
    >>> map_filename_to_vmaf('/media/mp4/joint_av_quality_v1/reference/IpMan2__frm57060to57300__SdrVp9Eve2pVE__1920_1080__7500_audio_kbps_192_videoChunksizeInSeconds_10_vmat99.51_phonevmaf100.00_psnr38.85_mbps7292.57.mp4')  # noqa E501
    """
    basename = os.path.basename(path)
    basename_no_ext = os.path.splitext(basename)[0]
    mo = re.search(r"_vmaf([0-9.]+)", basename_no_ext)
    if mo is None:
        return None
    vmaf = float(mo.group(1))
    return vmaf


def map_filename_to_phonevmaf(path: str) -> Optional[float]:
    """
    >>> map_filename_to_phonevmaf('/media/mp4/joint_av_quality_v1/MasterOfNoneS1E2__frm20000to20252__SdrVp9Eve2pVE__608_342__100_videoChunksizeInSeconds_10_vmaf22.85_phonevmaf37.40_psnr28.98_kbps98.74.mp4')  # noqa E501
    37.4
    >>> map_filename_to_phonevmaf('/media/mp4/joint_av_quality_v1/reference/IpMan2__frm57060to57300__SdrVp9Eve2pVE__1920_1080__7500_audio_kbps_192_videoChunksizeInSeconds_10_vmaf99.51_phonevmaf100.00_psnr38.85_kbps7292.57.mp4')  # noqa E501
    100.0
    >>> map_filename_to_phonevmaf('/media/mp4/joint_av_quality_v1/reference/IpMan2__frm57060to57300__SdrVp9Eve2pVE__1920_1080__7500_audio_kbps_192_videoChunksizeInSeconds_10_vmaf99.51_phonevmat100.00_psnr38.85_mbps7292.57.mp4')  # noqa E501
    """
    basename = os.path.basename(path)
    basename_no_ext = os.path.splitext(basename)[0]
    mo = re.search(r"_phonevmaf([0-9.]+)", basename_no_ext)
    if mo is None:
        return None
    phonevmaf = float(mo.group(1))
    return phonevmaf


def map_filename_to_qp(path: str) -> Optional[float]:
    """
    >>> map_filename_to_qp('/media/mp4/joint_av_quality_v1/MasterOfNoneS1E2__frm20000to20252__SdrVp9Eve2pVE__608_342__100_videoChunksizeInSeconds_10_vmaf22.85_phonevmaf37.40_psnr28.98_kbps98.74.mp4')  # noqa E501
    >>> map_filename_to_qp('ELE1NX122021/test2/training/taxi-driver-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp34_vmaf75.97_psnr33.34_kbps1774.05_fps23.976023976023978.mp4')  # noqa E501
    34
    >>> map_filename_to_qp('ELE1NX122021/test2/training/taxi-driver-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp31_vmaf81.11_psnr33.89_kbps2924.92_fps23.976023976023978.mp4')  # noqa E501
    31
    """
    basename = os.path.basename(path)
    basename_no_ext = os.path.splitext(basename)[0]
    mo = re.search(r"_qp([0-9]+)", basename_no_ext)
    if mo is None:
        return None
    qp = int(mo.group(1))
    return qp


def map_filename_to_title(path: str, length: int = 4) -> str:
    """
    >>> map_filename_to_title('/media/mp4/joint_av_quality_v1/MasterOfNoneS1E2__frm20000to20252__SdrVp9Eve2pVE__608_342__100_videoChunksizeInSeconds_10_vmaf22.85_phonevmaf37.40_psnr28.98_kbps98.74.mp4')  # noqa E501
    'Mast'
    >>> map_filename_to_title('ELE1NX122021/test2/training/taxi-driver-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp34_vmaf75.97_psnr33.34_kbps1774.05_fps23.976023976023978.mp4')  # noqa E501
    'taxi'
    >>> map_filename_to_title('ELE1NX122021/test2/training/taxi-driver-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp31_vmaf81.11_psnr33.89_kbps2924.92_fps23.976023976023978.mp4')  # noqa E501
    'taxi'
    >>> map_filename_to_title('/media/mp4/joint_av_quality_v1/reference/IpMan2__frm57060to57300__SdrVp9Eve2pVE__1920_1080__7500_audio_kbps_192_videoChunksizeInSeconds_10_vmaf99.51_phonevmaf100.00_psnr38.85_kbps7292.57.mp4')  # noqa E501
    'IpMa'
    >>> map_filename_to_title('/media/mp4/joint_av_quality_v1/reference/IpMan2__frm57060to57300__SdrVp9Eve2pVE__1920_1080__7500_audio_kbps_192_videoChunksizeInSeconds_10_vmaf99.51_phonevmaf100.00_psnr38.85_kbps7292.57.mp4', 6)  # noqa E501
    'IpMan2'
    """
    basename = os.path.basename(path)
    basename_no_ext = os.path.splitext(basename)[0]
    return basename_no_ext[:length]


def map_filename_to_height(path: str) -> Optional[float]:
    """
    >>> map_filename_to_height('/media/mp4/joint_av_quality_v1/MasterOfNoneS1E2__frm20000to20252__SdrVp9Eve2pVE__608_342__100_videoChunksizeInSeconds_10_vmaf22.85_phonevmaf37.40_psnr28.98_kbps98.74.mp4')  # noqa E501
    342
    >>> map_filename_to_height('/media/mp4/joint_av_quality_v1/reference/IpMan2__frm57060to57300__SdrVp9Eve2pVE__1920_1080__7500_audio_kbps_192_videoChunksizeInSeconds_10_vmaf99.51_phonevmaf100.00_psnr38.85_kbps7292.57.mp4')  # noqa E501
    1080
    >>> map_filename_to_height('/media/mp4/joint_av_quality_v1/reference/IpMan2__frm57060to57300__SdrVp9Eve2pVE__1920_1080__7500_audio_kbps_192_videoChunksizeInSeconds_10_vmaf99.51_phonevmat100.00_psnr38.85_mbps7292.57.mp4')  # noqa E501
    1080
    >>> map_filename_to_height('/media/mp4/joint_av_quality_v1/MasterOfNoneS1E2__frm20000to20252__SdrVp9Eve2pVE__608_342__100_videoChunksizeInSeconds_10_vmaf22.85_phonevmaf37.40_psnr28.98_kbps98.74.mp4')  # noqa E501
    342
    >>> map_filename_to_height('ELE1NX122021/test2/training/taxi-driver-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp34_vmaf75.97_psnr33.34_kbps1774.05_fps23.976023976023978.mp4')  # noqa E501
    2160
    >>> map_filename_to_height('ELE1NX122021/test2/training/taxi-driver-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160_none__qp31_vmaf81.11_psnr33.89_kbps2924.92_fps23.976023976023978.mp4')  # noqa E501
    """
    basename = os.path.basename(path)
    basename_no_ext = os.path.splitext(basename)[0]
    mo = re.search(r"__([0-9]+)_([0-9]+)__", basename_no_ext)
    if mo is None:
        return None
    height = int(mo.group(2))
    return height


def get_cmap(n, name='hsv'):
    """
    Returns a function that maps each index in 0, 1, ..., n-1 to a distinct
    RGB color; the keyword argument name must be a standard mpl colormap name.
    """
    return plt.cm.get_cmap(name, n)


def validate_xml(html):
    """
    Return True if html is a valid xml; otherwise return false

    >>> validate_xml(" <p> You have completed the test. Please take a moment to complete a post-test survey at: <a href=\\"https://forms.gle/oEBBS52D2BHeuCTN8\\" target=\\"_blank\\" rel=\\"noopener noreferrer\\"> https://forms.gle/oEBBS52D2BHeuC </a>.</p> ")  # noqa E501
    True
    >>> validate_xml(" <p> You have completed the test. Please take a moment to complete a post-test survey at: <a href=\\"https://forms.gle/oEBBS52D2BHeuCTN8\\" target=\\"_blank\\" rel=\\"noopener noreferrer\\"> https://forms.gle/oEBBS52D2BHeuC </a>.</pp> ")  # noqa E501
    False

    """
    try:
        etree.parse(StringIO(html), etree.HTMLParser(recover=False))
    except etree.XMLSyntaxError:
        return False
    return True

nest_site/nest/helpers.py
Netflix/e2nest
import json
import logging
import os
import random
import string
from typing import List, Optional

import numpy as np
import pandas as pd
from django.contrib.auth.models import User
from django.db import transaction
from django.db.utils import IntegrityError
from nest.config import ExperimentConfig, NestConfig, StimulusConfig
from nest.control import ExperimentController, SessionStatus
from nest.helpers import empty_object, map_path_to_noise_rmse, override
from nest.models import Content, DiscreteVote, Experiment, Experimenter, ExperimentRegister, Round, \
    Session, Stimulus, StimulusGroup, StimulusVoteGroup, Subject, Vote, VoteRegister
from nest.pages import CcrPage, map_methodology_to_page_class
from nest.sites import NestSite
from sureal.dataset_reader import PairedCompDatasetReader as \
    SurealPairedCompDatasetReader
from sureal.dataset_reader import RawDatasetReader as SurealRawDatasetReader

logging.basicConfig()
logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0])
logger.setLevel('INFO')


class SurealMixin(object):
    def _retrieve_or_create_stimuli(self) -> List[Stimulus]:
        dict_contentid_refvideo = dict()
        for ref_video in self.dataset.ref_videos:
            dict_contentid_refvideo[ref_video['content_id']] = ref_video
        stimuli = []
        for dis_video in self.dataset.dis_videos:
            ref_video = dict_contentid_refvideo[dis_video['content_id']]
            content = Content.objects.get(
                name=f"{self.dataset.dataset_name}-{ref_video['content_name']}",
                content_id=dis_video['content_id'])
            try:
                stimulus = Stimulus.objects.get(
                    content=content, stimulus_id=dis_video['asset_id'])
            except Stimulus.DoesNotExist:
                stimulus = Stimulus.objects.create(
                    content=content, stimulus_id=dis_video['asset_id'])
            stimuli.append(stimulus)
        return stimuli

    @staticmethod
    def find_or_create_experimenter(experiment, experimenter_username):
        # find or create user from experimenter's username
        user: User
        try:
            user = User.objects.get(username=experimenter_username)
            if not user.is_staff:
                raise AssertionError(
                    f"Cannot populate NEST with experimenter: user "
                    f"{experimenter_username} exists but is not staff")
        except User.DoesNotExist:
            user = User.objects.create_user(experimenter_username,
                                            password="", is_staff=True)
        # find or create experimenter from user
        experimenter: Experimenter
        try:
            experimenter = Experimenter.objects.get(user=user)
        except Experimenter.DoesNotExist:
            experimenter = Experimenter.objects.create(user=user)
        ExperimentRegister.objects.create(experiment=experiment,
                                          experimenter=experimenter)

    @staticmethod
    def make_experiment_config_dict(contents, stimuli, stimulusvotegroups, stimulusgroups, seed):
        d = dict()
        d['stimulus_config'] = {
            'contents': contents,
            'stimuli': stimuli,
            'stimulusvotegroups': stimulusvotegroups,
            'stimulusgroups': stimulusgroups,
        }
        additions = list()

        title = 'Instruction'
        text_html = ""
        text_html += """ <p> In this session, you will be asked to evaluate the annoyance caused by presence of banding artifact in a set of videos. </p> """  # noqa E501
        text_html += """ <p> If you don't know what banding artifacts are, it would be good idea to contact the author <b>ptandon@catflix.com</b> before proceeding with the test. </p> """  # noqa E501
        text_html += """ <p> There are two phases, a training and a testing phase. In the training, you will see 6 videos and will be asked to rate the presence of banding artifacts. The videos are chosen to span the scale and there is an expected score for each of the video to provide an anchor for the testing videos. </p> """  # noqa E501
        text_html += """ <p> Next, the testing phase will start and at this point, your scores will be recorded.</b> </p> """  # noqa E501
        text_html += """ <p> To play a video, click on the <b>"Play Video"</b> button. </p> """  # noqa E501
        text_html += """ <p> <b style='color:red !important;'>Please, maintain the viewing distance of approximately 1.5 times the monitor height.</b> </p> """  # noqa E501
        text_html += """ <p> Video clips will loop over, <b>press "space" key or double click mouse </b> to terminate the video. A slider will appear and you will be able to rate annoyance caused by the presence of banding in the video on the 0-100 scale, qualitatively from unwatchable to imperceptible. Please try to focus only on the banding artifacts. For all clips rate the worst case banding seen during video playback, even for videos with multiple scenes. </p>  """  # noqa E501
        text_html += """ <p> Once you have scored the video, click the "Next" button to fetch next video and repeat. </p> """  # noqa E501
        text_html += """ <p> Progress is shown for training and testing cases on top, near instructions. </p> """  # noqa E501
        actions_html = \
            """
            <p>
                <a class="button" href="{action_url}"  id="start">Start Test</a>
            </p>
            """
        additions.append({
            'position': {
                'round_id': 0,
                'before_or_after': 'before'
            },
            'context': {
                'title': title,
                'text_html': text_html,
                'actions_html': actions_html,
            }
        })
        d['experiment_config'] = {
            'title': 'test',
            'description': 'test',
            "methodology": "acr",  # hardcoded, not meant for actual test
            'vote_scale': 'FIVE_POINT',  # hardcoded, not meant for actual test
            'rounds_per_session': len(stimulusgroups),
            'random_seed': seed,
            'prioritized': [],
            'blocklist_stimulusgroup_ids': [],
            'additions': additions,
        }
        # assert conformity
        scfg = StimulusConfig(d['stimulus_config'], skip_path_check=True)
        _ = ExperimentConfig(stimulus_config=scfg, config=d['experiment_config'])
        return d


class SurealRawDatasetReaderPlus(SurealRawDatasetReader, SurealMixin):
    """
    Extended dataset reader that handles Sureal dataset format, and have extra
    capability such as populate NEST databases.
    """

    @override(SurealRawDatasetReader)
    def _assert_dataset(self):
        super()._assert_dataset()

        assert hasattr(self.dataset, 'dataset_name')
        assert hasattr(self.dataset, 'ref_videos')
        assert isinstance(self.dataset.ref_videos, list)
        for ref_video in self.dataset.ref_videos:
            assert 'content_name' in ref_video
            assert 'content_id' in ref_video
            assert 'path' in ref_video
        assert hasattr(self.dataset, 'dis_videos')
        assert isinstance(self.dataset.dis_videos, list)
        self.os_style = None
        for dis_video in self.dataset.dis_videos:
            assert 'asset_id' in dis_video
            assert 'content_id' in dis_video
            assert 'os' in dis_video

            assert (isinstance(dis_video['os'], list) or
                    isinstance(dis_video['os'], dict))

            if isinstance(dis_video['os'], list):
                assert len(dis_video['os']) == self.num_observers
                if self.os_style is None:
                    self.os_style = list
                else:
                    assert self.os_style == list
            elif isinstance(dis_video['os'], dict):
                if self.os_style is None:
                    self.os_style = dict
                else:
                    assert self.os_style == dict
            else:
                assert False

            assert 'path' in dis_video

    @transaction.atomic  # noqa C901
    def populate_nest(self, **more):  # noqa C901

        assert 'vote_class' in more
        vote_class = more['vote_class']

        experimenter_username = more['experimenter_username'] \
            if 'experimenter_username' in more else None
        assert experimenter_username is None or \
               isinstance(experimenter_username, str)

        # create experiment, dataset
        try:
            experiment = Experiment(title=self.dataset.dataset_name)
            experiment.save()
        except IntegrityError:
            raise AssertionError(
                f'Experiment with title {self.dataset.dataset_name} exists.')

        if experimenter_username is not None:
            self.find_or_create_experimenter(experiment, experimenter_username)

        # create contents
        for ref_video in self.dataset.ref_videos:
            Content(
                name=f"{self.dataset.dataset_name}-{ref_video['content_name']}",
                content_id=ref_video['content_id']).save()

        # create subjects and sessions
        if self.os_style == list:
            subjects = []
            sessions = []
            for subject_id in range(self.num_observers):
                subject = Subject(
                    name=f'{self.dataset.dataset_name}-#{subject_id}')
                subject.save()
                subjects.append(subject)

                session = Session(experiment=experiment, subject=subject)
                session.save()
                sessions.append(session)
        elif self.os_style == dict:
            subjects = []
            sessions = []
            for subject_name in self._get_list_observers():
                subject = Subject(
                    name=f'{self.dataset.dataset_name}-{subject_name}')
                subject.save()
                subjects.append(subject)

                session = Session(experiment=experiment, subject=subject)
                session.save()
                sessions.append(session)
        else:
            assert False

        # create stimuli
        stimuli = self._retrieve_or_create_stimuli()

        # create sg, svg
        sg_svg_cache = dict()
        for dis_video, stimulus in zip(self.dataset.dis_videos, stimuli):
            if stimulus.id in sg_svg_cache:
                # sg, svg = sg_svg_cache[stimulus.id]
                pass
            else:
                sg = StimulusGroup(stimulusgroup_id=stimulus.stimulus_id,
                                   experiment=experiment)
                sg.save()
                svg = StimulusVoteGroup.create_stimulusvotegroup_from_stimulus(
                    stimulus,
                    stimulusgroup=sg,
                    stimulusvotegroup_id=stimulus.stimulus_id,
                    experiment=experiment)
                sg_svg_cache[stimulus.id] = (sg, svg)

        # create rounds
        for rid, (dis_video, stimulus) in enumerate(zip(self.dataset.dis_videos, stimuli)):
            assert len(subjects) == len(sessions)
            for session_id, session in enumerate(sessions):
                assert stimulus.id in sg_svg_cache
                sg, svg = sg_svg_cache[stimulus.id]
                rnd = Round(session=session, stimulusgroup=sg,
                            round_id=rid,
                            )
                rnd.save()

        # create votes
        for rid, (dis_video, stimulus) in enumerate(zip(self.dataset.dis_videos, stimuli)):
            os = dis_video['os']
            if self.os_style == list:
                assert isinstance(os, list)
                assert len(os) == len(subjects) == len(sessions)
                for score, session in zip(os, sessions):
                    if stimulus.id in sg_svg_cache:
                        sg, svg = sg_svg_cache[stimulus.id]
                    else:
                        sg = StimulusGroup(stimulusgroup_id=stimulus.stimulus_id,
                                           experiment=experiment)
                        sg.save()
                        svg = StimulusVoteGroup.create_stimulusvotegroup_from_stimulus(
                            stimulus,
                            stimulusgroup=sg,
                            stimulusvotegroup_id=stimulus.stimulus_id,
                            experiment=experiment)
                        sg_svg_cache[stimulus.id] = (sg, svg)

                    rnd = Round.objects.get(session=session, stimulusgroup=sg,
                                            round_id=rid,
                                            )
                    vote = vote_class(score=score,
                                      round=rnd,
                                      stimulusvotegroup=svg)
                    vote.save()

            elif self.os_style == dict:
                assert isinstance(os, dict)
                for subject_name in os.keys():
                    for subject in subjects:
                        if subject.name == f'{self.dataset.dataset_name}-{subject_name}':
                            break
                    else:
                        assert False, \
                            f'no subject matches name {self.dataset.dataset_name}-{subject_name}'
                    session = Session.objects.get(experiment=experiment,
                                                  subject=subject)

                    if stimulus.id in sg_svg_cache:
                        sg, svg = sg_svg_cache[stimulus.id]
                    else:
                        sg = StimulusGroup(stimulusgroup_id=stimulus.stimulus_id,
                                           experiment=experiment)
                        sg.save()
                        svg = StimulusVoteGroup.create_stimulusvotegroup_from_stimulus(
                            stimulus,
                            stimulusgroup=sg,
                            stimulusvotegroup_id=stimulus.stimulus_id,
                            experiment=experiment)

                        sg_svg_cache[stimulus.id] = (sg, svg)

                    rnd = Round.objects.get(session=session, stimulusgroup=sg,
                                            round_id=rid,
                                            )
                    vote = vote_class(score=os[subject_name],
                                      round=rnd,
                                      stimulusvotegroup=svg)
                    vote.save()

            else:
                assert False

    def get_stimulus_dict(self):
        contents = list()
        for ref_video in self.dataset.ref_videos:
            contents.append({
                'content_id': ref_video['content_id'],
                'name': ref_video['content_name'],
            })
        stimuli = list()
        stimulusvotegroups = list()
        stimulusgroups = list()
        for dis_video in self.dataset.dis_videos:
            stimuli.append({
                'path': dis_video['path'],
                'stimulus_id': dis_video['asset_id'],
                'type': 'video/mp4',  # hardcoded, don't use in real test config
                'content_id': dis_video['content_id'],
            })
            stimulusvotegroups.append({
                'stimulus_ids': [dis_video['asset_id']],
                'stimulusvotegroup_id': dis_video['asset_id'],
            })
            stimulusgroups.append({
                'stimulusvotegroup_ids': [dis_video['asset_id']],
                'stimulusgroup_id': dis_video['asset_id'],
            })
        return {'contents': contents,
                'stimuli': stimuli,
                'stimulusvotegroups': stimulusvotegroups,
                'stimulusgroups': stimulusgroups,
                }


class SurealPairedCompDatasetReaderPlus(SurealPairedCompDatasetReader, SurealMixin):
    """
    Extended paired comparison dataset reader that handles Sureal dataset
    format, and have extra capability such as populate NEST databases.
    """

    @override(SurealPairedCompDatasetReader)
    def _assert_dataset(self):
        super()._assert_dataset()

        assert hasattr(self.dataset, 'dataset_name')
        assert hasattr(self.dataset, 'ref_videos')
        assert isinstance(self.dataset.ref_videos, list)
        for ref_video in self.dataset.ref_videos:
            assert 'content_name' in ref_video
            assert 'content_id' in ref_video
            assert 'path' in ref_video
        assert hasattr(self.dataset, 'dis_videos')
        assert isinstance(self.dataset.dis_videos, list)
        self.os_style = None
        for dis_video in self.dataset.dis_videos:
            assert 'asset_id' in dis_video
            assert 'content_id' in dis_video
            assert 'os' in dis_video
            assert isinstance(dis_video['os'], dict)
            for key in dis_video['os'].keys():
                assert isinstance(key[0], str)
                assert isinstance(key[1], int)
            assert 'path' in dis_video

    @transaction.atomic
    def populate_nest(self, **more):

        assert 'vote_class' in more
        vote_class = more['vote_class']

        experimenter_username = more['experimenter_username'] \
            if 'experimenter_username' in more else None
        assert experimenter_username is None or \
               isinstance(experimenter_username, str)

        # create experiment, dataset
        try:
            experiment = Experiment(title=self.dataset.dataset_name)
            experiment.save()
        except IntegrityError:
            raise AssertionError(
                f'Experiment with title {self.dataset.dataset_name} exists.')

        if experimenter_username is not None:
            self.find_or_create_experimenter(experiment, experimenter_username)

        # create contents
        for ref_video in self.dataset.ref_videos:
            Content(
                name=f"{self.dataset.dataset_name}-{ref_video['content_name']}",
                content_id=ref_video['content_id']).save()

        # create subjects and sessions
        subjects = []
        sessions = []
        for subject_name in self._get_list_observers():
            subject = Subject(name=f'{self.dataset.dataset_name}-{subject_name}')
            subject.save()
            subjects.append(subject)

            session = Session(experiment=experiment, subject=subject)
            session.save()
            sessions.append(session)

        # create stimuli
        stimuli = self._retrieve_or_create_stimuli()

        # create rounds, sg, svg, votes
        d_subjname_to_rid = dict()
        sg_svg_cache = dict()
        sgid_count = 0
        for dis_video, stimulus in zip(self.dataset.dis_videos, stimuli):
            os = dis_video['os']
            assert isinstance(os, dict)
            for key in os.keys():
                subject_name, pc_stim_id = key
                for subject in subjects:
                    if subject.name == f'{self.dataset.dataset_name}-{subject_name}':
                        break
                else:
                    assert False, \
                        f'no subject matches name {self.dataset.dataset_name}-{subject_name}'
                session = Session.objects.get(
                    experiment=experiment, subject=subject)
                for stimulus_against in stimuli:
                    if stimulus_against.stimulus_id == pc_stim_id:
                        break
                else:
                    assert False, \
                        f'no stimulus matches stimulus_id {pc_stim_id}'

                if (stimulus.id, stimulus_against.id) in sg_svg_cache:
                    sg, svg = sg_svg_cache[(stimulus.id, stimulus_against.id)]
                else:
                    sg = StimulusGroup(stimulusgroup_id=sgid_count,
                                       experiment=experiment)
                    sg.save()
                    sgid_count += 1
                    svg = StimulusVoteGroup.create_stimulusvotegroup_from_stimuli_pair(
                        stimulus, stimulus_against,
                        stimulusgroup=sg,
                        stimulusvotegroup_id=sgid_count,
                        experiment=experiment)
                    sg_svg_cache[(stimulus.id, stimulus_against.id)] = (sg, svg)

                rnd = Round(session=session, stimulusgroup=sg,
                            round_id=d_subjname_to_rid.setdefault(session.subject.name, 0))
                d_subjname_to_rid[session.subject.name] += 1
                rnd.save()
                vote = vote_class(score=os[key],
                                  round=rnd,
                                  stimulusvotegroup=svg)
                vote.save()

    @property
    def ref_score(self):
        raise NotImplementedError

    @property
    def opinion_score_2darray(self):
        # return np.nansum(self.opinion_score_3darray, axis=1)
        raise NotImplementedError

    def to_persubject_dataset(self, quality_scores, **kwargs):
        raise NotImplementedError

    def get_stimulusgroups(self):
        raise NotImplementedError


def import_sureal_dataset(dataset, scale, **more):
    vote_class = Vote.find_subclass(scale)
    try:
        dataset_reader = SurealPairedCompDatasetReaderPlus(dataset)
    except AssertionError:
        dataset_reader = SurealRawDatasetReaderPlus(dataset)

    dataset_reader.populate_nest(vote_class=vote_class, **more)

    write_config = more['write_config'] if 'write_config' in more else False
    assert isinstance(write_config, bool)
    if write_config:
        config_filedir = more['config_filedir'] if 'config_filedir' in more \
            else NestConfig.media_path('experiment_config')
        sd = dataset_reader.get_stimulus_dict()
        assert 'contents' in sd
        assert 'stimuli' in sd
        assert 'stimulusvotegroups' in sd
        assert 'stimulusgroups' in sd
        config = dataset_reader.make_experiment_config_dict(
            sd['contents'],
            sd['stimuli'],
            sd['stimulusvotegroups'],
            sd['stimulusgroups'],
            seed=None
        )
        config_filepath = os.path.join(
            config_filedir, f"{dataset_reader.dataset.dataset_name}.json")
        os.makedirs(os.path.dirname(config_filepath), exist_ok=True)
        with open(config_filepath, "wt") as fp:
            json.dump(config, fp, indent=4)


def export_sureal_dataset(experiment_title, ignore_against: bool = False):  # noqa C901
    """
    ignore_against: set True if the experiment methodology is DCR or SAMVIQ,
    with which Sureal format does not have Reference as "against"
    """
    dataset = empty_object()

    experiment = Experiment.objects.get(title=experiment_title)

    try:
        config = NestSite._load_experiment_config2(experiment_title, is_test=False)
        ec = NestSite._get_experiment_controller2(experiment, config)
    except FileNotFoundError:
        ec = None

    sessions = Session.objects.filter(experiment=experiment).all()

    ref_video_dict = dict()  # dict: content_id -> ref_video
    dis_video_dict = dict()  # dict: asset_id -> dis_video

    os_dict_style = None  # either
    for session in sessions:

        # subject_name logic
        subject = session.subject
        subject_name = subject.get_subject_name()

        for rnd in session.round_set.all():
            if rnd.vote_set.count() == 0:
                continue
            vote: Vote
            for vote in rnd.vote_set.all():
                svg = StimulusVoteGroup.objects.get(vote=vote)
                vrs = VoteRegister.objects.filter(stimulusvotegroup=svg)
                assert vrs.count() == 1 or vrs.count() == 2, \
                    f'expect #stimuli to be either 1 or 2, but get {vrs.count()}'
                if vrs.count() == 1:
                    if os_dict_style is None:
                        os_dict_style = 'single'
                    else:
                        assert os_dict_style == 'single'

                    assert ignore_against is False, \
                        'cannot have os_dict_style single AND ignore_against ' \
                        'is False: there is nothing against'

                    vr = vrs.all()[0]
                    stimulus: Stimulus = vr.stimulus
                    content: Content = stimulus.content
                    if content.content_id not in ref_video_dict:
                        ref_video = {
                            'content_id': content.content_id,
                            'content_name': content.name,
                            'path': str(content.content_id),
                        }
                        ref_video_dict[content.content_id] = ref_video

                    if stimulus.stimulus_id not in dis_video_dict:
                        if ec is not None:
                            stim_dict = NestSite._get_matched_stimulus_dict(
                                ec, stimulus.stimulus_id)
                            path = stim_dict['path']
                        else:
                            path = str(stimulus.stimulus_id)
                        dis_video = {
                            'asset_id': stimulus.stimulus_id,
                            'content_id': content.content_id,
                            'os': dict(),
                            'path': path,
                        }
                        dis_video_dict[stimulus.stimulus_id] = dis_video

                    if subject_name in dis_video_dict[stimulus.stimulus_id]['os']:
                        if not isinstance(dis_video_dict[stimulus.stimulus_id]['os'][subject_name], list):
                            dis_video_dict[stimulus.stimulus_id]['os'][subject_name] = \
                                [dis_video_dict[stimulus.stimulus_id]['os'][subject_name]]
                        dis_video_dict[stimulus.stimulus_id]['os'][subject_name].append(vote.score)
                    else:
                        dis_video_dict[stimulus.stimulus_id]['os'][subject_name] = vote.score

                elif vrs.count() == 2:
                    if os_dict_style is None:
                        os_dict_style = 'double'
                    else:
                        assert os_dict_style == 'double'

                    vrs = vrs.order_by('stimulus_order')
                    assert vrs[0].stimulus_order == 1
                    assert vrs[1].stimulus_order == 2
                    stimulus = vrs[0].stimulus
                    stimulus_against = vrs[1].stimulus
                    content = stimulus.content
                    content_against = stimulus_against.content
                    if content.content_id not in ref_video_dict:
                        ref_video = {
                            'content_id': content.content_id,
                            'content_name': content.name,
                            'path': str(content.content_id),
                        }
                        ref_video_dict[content.content_id] = ref_video

                    if stimulus.stimulus_id not in dis_video_dict:
                        if ec is not None:
                            stim_dict = NestSite._get_matched_stimulus_dict(
                                ec, stimulus.stimulus_id)
                            path = stim_dict['path']
                        else:
                            path = str(stimulus.stimulus_id)
                        dis_video = {
                            'asset_id': stimulus.stimulus_id,
                            'content_id': content.content_id,
                            'os': dict(),
                            'path': path,
                        }
                        dis_video_dict[stimulus.stimulus_id] = dis_video

                    if stimulus_against.stimulus_id not in dis_video_dict:
                        if ec is not None:
                            stim_dict = NestSite._get_matched_stimulus_dict(
                                ec, stimulus_against.stimulus_id)
                            path = stim_dict['path']
                        else:
                            path = str(stimulus_against.stimulus_id)
                        dis_video = {
                            'asset_id': stimulus_against.stimulus_id,
                            'content_id': content_against.content_id,
                            'os': dict(),
                            'path': path,
                        }
                        dis_video_dict[stimulus_against.stimulus_id] = dis_video

                    if ignore_against is True:
                        if subject_name in dis_video_dict[stimulus.stimulus_id]['os']:
                            if not isinstance(dis_video_dict[stimulus.stimulus_id]['os'][subject_name], list):
                                dis_video_dict[stimulus.stimulus_id]['os'][subject_name] = \
                                    [dis_video_dict[stimulus.stimulus_id]['os'][subject_name]]
                            dis_video_dict[stimulus.stimulus_id]['os'][subject_name].append(vote.score)
                        else:
                            dis_video_dict[stimulus.stimulus_id]['os'][subject_name] = vote.score
                    else:
                        if (subject_name, stimulus_against.stimulus_id) in dis_video_dict[stimulus.stimulus_id]['os']:
                            if not isinstance(dis_video_dict[stimulus.stimulus_id]['os'][subject_name, stimulus_against.stimulus_id], list):
                                dis_video_dict[stimulus.stimulus_id]['os'][subject_name, stimulus_against.stimulus_id] = \
                                    [dis_video_dict[stimulus.stimulus_id]['os'][subject_name, stimulus_against.stimulus_id]]
                            dis_video_dict[stimulus.stimulus_id]['os'][subject_name, stimulus_against.stimulus_id].append(vote.score)
                        else:
                            dis_video_dict[stimulus.stimulus_id]['os'][subject_name, stimulus_against.stimulus_id] = vote.score

                else:
                    assert False

    dataset.dataset_name = experiment.title
    dataset.ref_videos = [ref_video_dict[content_id]
                          for content_id in sorted(ref_video_dict.keys())]
    dataset.dis_videos = [dis_video_dict[asset_id]
                          for asset_id in sorted(dis_video_dict.keys())]

    return dataset


class ExperimentUtils(object):

    @staticmethod
    def get_experiment_controller(experiment_title: str,
                                  config: dict = None,
                                  skip_path_check: bool = False) -> ExperimentController:
        if config is None:
            with open(NestConfig.media_path('experiment_config',
                                            f"{experiment_title}.json"),
                      "rt") as fp:
                config = json.load(fp)
        exp = Experiment.objects.get(title=experiment_title)
        assert isinstance(config, dict)
        assert 'stimulus_config' in config
        assert 'experiment_config' in config
        scfg = StimulusConfig(config['stimulus_config'],
                              skip_path_check=skip_path_check)
        ecfg = ExperimentConfig(stimulus_config=scfg,
                                config=config['experiment_config'])
        ec = ExperimentController(experiment=exp, experiment_config=ecfg)
        return ec

    @staticmethod
    def _create_experiment_from_config(
            source_config_filepath: str,
            is_test: bool = False,
            random_seed: int = None,
            experiment_title: str = None):

        if is_test:
            assert random_seed is not None and isinstance(random_seed, int)
            assert experiment_title is not None and isinstance(experiment_title, str)
        else:
            if random_seed is not None:
                assert isinstance(random_seed, int)
            if experiment_title is not None:
                assert isinstance(experiment_title, str)

        if experiment_title is None:
            with open(source_config_filepath, 'rt') as fp_source:
                config = json.load(fp_source)
            experiment_title = config['experiment_config']['title']

        target_config_filepath = NestSite.get_experiment_config_filepath(
            experiment_title, is_test=is_test)
        assert target_config_filepath != source_config_filepath
        os.makedirs(os.path.dirname(target_config_filepath), exist_ok=True)
        with open(source_config_filepath, 'rt') as fp_source:
            with open(target_config_filepath, 'wt') as fp_target:
                config = json.load(fp_source)
                if is_test:
                    config['experiment_config']['random_seed'] = random_seed
                    config['experiment_config']['title'] = experiment_title
                else:
                    if random_seed is not None:
                        config['experiment_config']['random_seed'] = random_seed
                    if experiment_title is not None:
                        config['experiment_config']['title'] = experiment_title
                json.dump(config, fp_target, indent=4)
        with open(target_config_filepath, 'rt') as fp:
            config = json.load(fp)
        if is_test:
            scfg = StimulusConfig(config['stimulus_config'], skip_path_check=True)
        else:
            scfg = StimulusConfig(config['stimulus_config'])
        ecfg = ExperimentConfig(stimulus_config=scfg,
                                config=config['experiment_config'])
        e = Experiment(title=config['experiment_config']['title'],
                       description=config['experiment_config']['description'])
        e.save()
        ec = ExperimentController(experiment=e, experiment_config=ecfg)
        ec.populate_stimuli()
        return ec

    @classmethod
    def validate_config(cls,
                        experiment_config_filepath: str):
        """
        Validate if the experiment config file is a good one.
        """

        # do not check validity of these fields
        round_context_default_d = {
            'title': None,
            'video': None, 'video_a': None, 'video_b': None,
            'button': None, 'button_a': None, 'button_b': None,
            'videos': [None], 'video_ref': None,
            'buttons': [None], 'button_ref': None,
            'stimulusvotegroup_ids': [None],
            'stimulusvotegroup_id': 0,
        }

        with open(experiment_config_filepath, 'rt') as fp_source:
            config = json.load(fp_source)
        experiment_title = config['experiment_config']['title']
        try:
            Experiment.objects.get(title=experiment_title)
            raise AssertionError(
                f"Experiment with title {experiment_title} already exists.")
        except Experiment.DoesNotExist:
            pass
        scfg = StimulusConfig(config['stimulus_config'])
        ecfg = ExperimentConfig(
            stimulus_config=scfg,
            config=config['experiment_config'])

        PageClass = map_methodology_to_page_class(ecfg.methodology)
        VoteClass = Vote.find_subclass(ecfg.vote_scale)

        d = dict()
        d.update(round_context_default_d)
        if PageClass == CcrPage:
            assert issubclass(VoteClass, DiscreteVote), \
                f"expect VoteClass to be subclass of DiscreteVote, but is: {VoteClass}"
            d['video_a_to_b_values'] = VoteClass.support
        d.update(ecfg.round_context)

        _ = PageClass(d)

        if 'choices' in d and issubclass(VoteClass, DiscreteVote):
            assert len(d['choices']) == len(VoteClass.support), \
                f"the length of choices ({len(d['choices'])}) does not match " \
                f"vote_scale {ecfg.vote_scale}"

    @classmethod
    def create_experiment(cls,
                          experiment_config_filepath: str,
                          experimenter_username: str,
                          is_test: bool = False,
                          random_seed: int = None,
                          experiment_title: str = None,
                          ):
        """
        Create a new experiment from config file specified by
        experiment_config_filepath, with the experimenter associated with
        experimenter_username.
        """
        ec: ExperimentController = cls._create_experiment_from_config(
            source_config_filepath=experiment_config_filepath,
            is_test=is_test,
            random_seed=random_seed,
            experiment_title=experiment_title)

        exper = Experimenter.find_by_username(experimenter_username)
        if exper is None:
            exper = Experimenter.create_by_username(experimenter_username)
        ExperimentRegister.objects.create(experiment=ec.experiment,
                                          experimenter=exper)

        return ec

    @classmethod
    def delete_experiment_by_title(cls,
                                   experiment_title: str = None,
                                   is_test: bool = False,
                                   ):
        """
        Delete an experiment by experiment_title, through cleaning up the
        relevant objects in the DB and removing the config file.
        """
        target_config_filepath = NestSite.get_experiment_config_filepath(
            experiment_title, is_test=is_test)
        assert os.path.exists(target_config_filepath)
        exp: Experiment = Experiment.objects.get(title=experiment_title)
        exp.delete()
        os.remove(target_config_filepath)

    @classmethod
    def add_session_to_experiment(cls,
                                  experiment_title: str,
                                  subject_username: str,
                                  config: dict = None,
                                  skip_path_check: bool = False) -> Session:
        """
        Add a new session (Not started) to experiment with experiment_title,
        assigning to a subject with username. config default to None and should
        be read from a config file, and is only supplied for testing purpose.
        skip_path_check is True only for testing purpose.
        """
        ec = cls.get_experiment_controller(experiment_title, config, skip_path_check)

        subj = Subject.find_by_username(subject_username)
        if subj is None:
            subj = Subject.create_by_username(subject_username)
        return ec.add_session(subj)

    @classmethod
    def delete_session_by_id(cls,
                             session_id: int,
                             experiment_title: str,
                             config: dict = None,
                             skip_path_check: bool = False):
        """
        Delete a session by a session ID. config default to None and should
        be read from a config file, and is only supplied for testing purpose.
        skip_path_check is True only for testing purpose.
        """
        ec = cls.get_experiment_controller(experiment_title, config, skip_path_check)
        ec.delete_session(session_id)

    @classmethod
    def update_subject_for_session(cls,
                                   experiment_title: str,
                                   session_id: int,
                                   subject_username: str,
                                   config: dict = None,
                                   skip_path_check: bool = False):
        """
        Update the subject of the n-th session of an experiment with
        experiment_title, assigned to a subject with username. function
        is mainly for testing purpose.
        """
        ec = cls.get_experiment_controller(experiment_title, config,
                                           skip_path_check=skip_path_check)

        subj = Subject.find_by_username(subject_username)
        if subj is None:
            subj = Subject.create_by_username(subject_username)
        sess: Session = ec.experiment.session_set.get(id=session_id)
        sess.subject = subj
        sess.save()

    @classmethod
    def reset_unfinished_session(cls,
                                 experiment_title: str,
                                 session_id: int,
                                 config: dict = None,
                                 skip_path_check: bool = False):
        """
        Reset an unfinished session. config dict not None is only for testing
        purpose. skip_path_check True only for testing purpose.
        """
        ec: ExperimentController = \
            cls.get_experiment_controller(experiment_title,
                                          config, skip_path_check)
        sess: Session = ec.experiment.session_set.get(id=session_id)
        assert ec.get_session_status(sess) == SessionStatus.PARTIALLY_FINISHED
        return ec.reset_session(sess)


class ESUtilities(object):

    @staticmethod
    def export_playlist_ES_noise_study_style(
            ec: ExperimentController,
            shuffle_intervals: bool = False,
            methodology: Optional[str] = None,
    ) -> pd.DataFrame:

        assert methodology is None or methodology in ['samviq']

        stimuli_info = ec.get_stimuli_info()
        assert 'contents' in stimuli_info
        assert 'stimuli' in stimuli_info
        assert 'stimulusvotegroups' in stimuli_info
        assert 'stimulusgroups' in stimuli_info

        # build dict of dict:
        dd_s = dict(zip([s['stimulus_id'] for s in stimuli_info['stimuli']], stimuli_info['stimuli']))
        dd_svg = dict(zip([s['stimulusvotegroup_id'] for s in stimuli_info['stimulusvotegroups']], stimuli_info['stimulusvotegroups']))

        rows = list()
        session: Session
        for session in ec.experiment.session_set.all():
            sess_info: dict = ec.get_session_info(session)
            assert 'subject' in sess_info
            assert 'rounds' in sess_info
            rd: dict
            for rd in sess_info['rounds']:
                assert 'round_id' in rd
                assert 'stimulusvotegroups' in rd

                if methodology is None:
                    svgds: list = rd['stimulusvotegroups']

                    assert len(svgds) == 1, f'currently expect only one svg per round if methodology is None, but {svgds}'

                    svgd: dict
                    for svgd in svgds:  # prepared for generalization beyond len(svgds) == 1
                        assert 'stimulusvotegroup_id' in svgd
                        d_svg: dict = dd_svg[svgd['stimulusvotegroup_id']]
                        assert 'stimulus_ids' in d_svg
                        sids = d_svg['stimulus_ids']

                        if shuffle_intervals is True:
                            random.shuffle(sids)
                        for isid, sid in enumerate(sids):
                            d_s: dict = dd_s[sid]
                            assert 'path' in d_s
                            assert 'content_id' in d_s
                            filename = d_s['path']
                            noise_level = map_path_to_noise_rmse(os.path.basename(filename))

                            row = {
                                'participant_id': sess_info['subject'],
                                'content': d_s['content_id'],
                                'noise_level': noise_level,
                                'interval': string.ascii_uppercase[isid],
                                'trial': rd['round_id'],
                                'Filename': filename,
                            }
                            rows.append(row)
                elif methodology == 'samviq':
                    svgds: list = rd['stimulusvotegroups']
                    assert len(svgds) > 1
                    if shuffle_intervals is True:
                        random.shuffle(svgds)
                    svgd: dict
                    enc_sids = list()
                    ref_sid = None
                    for svgd in svgds:
                        assert 'stimulusvotegroup_id' in svgd
                        d_svg: dict = dd_svg[svgd['stimulusvotegroup_id']]
                        assert 'stimulus_ids' in d_svg
                        sids = d_svg['stimulus_ids']
                        assert len(sids) == 2
                        enc_sid_, ref_sid_ = sids
                        if ref_sid is None:
                            ref_sid = ref_sid_
                        else:
                            assert ref_sid == ref_sid_
                        enc_sids.append(enc_sid_)
                    assert ref_sid is not None
                    assert len(enc_sids) == len(svgds)
                    for isid, sid in enumerate(enc_sids + [ref_sid]):
                        d_s: dict = dd_s[sid]
                        assert 'path' in d_s
                        assert 'content_id' in d_s
                        filename = d_s['path']
                        noise_level = map_path_to_noise_rmse(os.path.basename(filename))

                        row = {
                            'participant_id': sess_info['subject'],
                            'content': d_s['content_id'],
                            'noise_level': noise_level,
                            'interval': string.ascii_uppercase[isid],
                            'trial': rd['round_id'],
                            'Filename': filename,
                        }
                        rows.append(row)
                else:
                    assert False
        return pd.DataFrame(rows)

    @staticmethod
    def visualize_playlist_ES_style(ax, csv_filepath: str):
        import pprint
        df = pd.read_csv(csv_filepath, sep=',')
        # print(df)
        enc_filepaths = sorted(set(df['Filename']))
        dict_enc_filepaths = dict(zip(enc_filepaths, range(len(enc_filepaths))))
        subjects = sorted(set(df['participant_id']))
        dict_subjects = dict(zip(subjects, range(len(subjects))))
        mtx = np.ones([len(enc_filepaths), len(subjects)])
        for index, row in df.iterrows():
            mtx[dict_enc_filepaths[row['Filename']], dict_subjects[row['participant_id']]] = 0
        ax.imshow(mtx, interpolation='nearest', cmap='gray')
        ax.set_title(os.path.basename(csv_filepath))
        ax.set_ylabel('Stimulus')
        ax.set_xlabel('Subject')
        pprint.pprint(dict_subjects)
        pprint.pprint(dict_enc_filepaths)


class UserUtils(object):

    @staticmethod
    def create_user(username: str, is_staff: bool = False, is_superuser: bool = False) -> User:
        try:
            user = User.objects.get(username=username)
            touched = False
            if user.is_staff is False and is_staff is True:
                user.is_staff = True
                touched = True
            if user.is_superuser is False and is_superuser is True:
                user.is_superuser = True
                touched = True
            if touched:
                user.save()
        except User.DoesNotExist:
            user = User.objects.create_user(username=username,
                                            is_staff=is_staff,
                                            is_superuser=is_superuser)
        return user

    @staticmethod
    def set_password(username: str, password: str):
        try:
            user = User.objects.get(username=username)
            user.set_password(password)
            user.save()
        except User.DoesNotExist:
            logger.warning(f'user {username} does not exist; skip set_password')

nest_site/nest/io.py
Netflix/e2nest
class ExperimentConfigFileUtils(object):

    @staticmethod
    def add_to_stimulusvotegroup_dict(stimulus_ids, d_stimulusvotegroups):
        # Note: since sg-to-svg is one-to-many, each svg corresponds to a
        # unique sg. Thus, we cannot reuse svg across different sgs, and
        # must create fresh svg_id every time.
        stimulusvotegroup_id = len(d_stimulusvotegroups)
        d_stimulusvotegroups[stimulusvotegroup_id] = {
            'stimulus_ids': stimulus_ids,
            'stimulusvotegroup_id': stimulusvotegroup_id}
        return stimulusvotegroup_id

    @staticmethod
    def add_to_stimulus_dict(filepath, content_id, d_stimuli):
        if filepath in d_stimuli:
            stimulus_id = d_stimuli[filepath]['stimulus_id']
        else:
            stimulus_id = len(d_stimuli)
            d_stimuli[filepath] = {
                'path': filepath,
                'stimulus_id': stimulus_id,
                'type': 'video/mp4',
                'content_id': content_id}
        return stimulus_id

    @staticmethod
    def add_to_content_dict(content_name, d_contents):
        if content_name in d_contents:
            content_id = d_contents[content_name]['content_id']
        else:
            content_id = len(d_contents)
        d_contents[content_name] = {
            'content_id': content_id,
            'name': content_name}
        return content_id

    @staticmethod
    def add_to_stimulusgroup_dict(stimulusvotegroup_ids, d_stimulusgroups):
        if str(stimulusvotegroup_ids) in d_stimulusgroups:
            stimulusgroup_id = d_stimulusgroups[str(stimulusvotegroup_ids)][
                'stimulusgroup_id']
        else:
            stimulusgroup_id = len(d_stimulusgroups)
            d_stimulusgroups[str(stimulusvotegroup_ids)] = {
                'stimulusvotegroup_ids': stimulusvotegroup_ids,
                'stimulusgroup_id': stimulusgroup_id}
        return stimulusgroup_id

    @staticmethod
    def populate_prioritized_etc(stimulusgroup_id, mode, prioritized,
                                 blocklist_stimulusgroup_ids, training_round_ids):
        if mode == 'training':
            round_id = len(prioritized)
            prioritized.append({
                'session_idx': None,
                'round_id': round_id,
                'stimulusgroup_id': stimulusgroup_id,
            })
            blocklist_stimulusgroup_ids.append(stimulusgroup_id)
            training_round_ids.append(round_id)
        elif mode == 'reliability':
            prioritized.append({
                'session_idx': None,
                'round_id': None,
                'stimulusgroup_id': stimulusgroup_id,
            })
            blocklist_stimulusgroup_ids.append(stimulusgroup_id)
        elif mode == 'testing':
            pass  # do nothing
        else:
            assert False

nest_site/nest/io_utils.py
Netflix/e2nest
from __future__ import annotations

from abc import ABCMeta, abstractmethod
from typing import List

from django.contrib.auth.models import User
from django.db import models, transaction
from django.utils import timezone
from polymorphic.models import PolymorphicModel

from .helpers import TypeVersionEnabled


class GenericModel(PolymorphicModel):
    """
    Abstract model that serves as the base class of other models derived.

    It has a created_date field that can be filled automatically with the the
    creation date, and a deactivate_date filed that is None by default,
    signaling that an object is active. Once deactivate_date is set to not None,
    it signals that the object is deactivated from the set date on.
    """
    create_date = models.DateTimeField('date created', default=timezone.now)
    deactivate_date = models.DateTimeField('date deactivated', null=True,
                                           default=None, blank=True)

    @property
    def is_active(self):
        """Returns if the object is currently active."""
        return (self.deactivate_date is None
                or timezone.now() < self.deactivate_date)

    def deactivate(self):
        """To deactivate the object."""
        self.deactivate_date = timezone.now()

    class Meta:
        abstract = True

    @property
    def saved(self):
        """Returns if the object is saved in database."""
        return not (self.pk is None)

    def __str__(self):
        return f"{self.__class__.__name__} {self.id}"


class Person(GenericModel):
    """Abtract class for a Person."""
    name = models.CharField('name', max_length=100, default="", blank=True)
    user: User = models.OneToOneField(User,
                                      null=True, blank=True,
                                      on_delete=models.SET_NULL)

    class Meta:
        abstract = True

    def __str__(self):
        s = super().__str__()
        disp_fields = []
        if self.name != "":
            disp_fields += [self.name]
        if self.user is not None:
            disp_fields += [str(self.user)]
        if len(disp_fields) > 0:
            s += f" ({', '.join(disp_fields)})"
        return s

    @classmethod
    def find_by_username(cls, username: str):
        try:
            user = User.objects.get(username=username)
        except User.DoesNotExist:
            return None
        try:
            person = cls.objects.get(user=user)
        except cls.DoesNotExist:
            return None
        return person


class Subject(Person):
    """Subject of an Experiment."""

    @classmethod
    @transaction.atomic
    def create_by_username(cls, username: str):
        """
        Create a new Subject object and associate it to a User of username.
        The method will try two things: 1) if the User exists, it will
        associate the new Subject to the User; 2) if the User does not exist,
        it will try to create a new User before association.
        """
        try:
            user = User.objects.get(username=username)
        except User.DoesNotExist:
            user = User.objects.create_user(username=username)
        assert cls.objects.filter(user=user).count() == 0
        p = cls.objects.create(user=user)
        p.save()
        return p

    def get_subject_name(self):
        subject_name = self.name
        if subject_name == "":
            if self.user is not None:
                subject_name = self.user.username
        if subject_name == "":
            subject_name = str(self.id)
        return subject_name


class Experimenter(Person):
    """Experimenter of an Experiment."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        assert self.user is None or (self.user is not None and self.user.is_staff)

    @classmethod
    @transaction.atomic
    def create_by_username(cls, username: str):
        """
        Create a new Experimenter object and associate it to a User of username.
        The User must be is_staff. The method will try three things: 1) if the
        User exists, and is_staff, it will associate the new Experimenter to the
        User; 2) if the User exist but not is_staff, it will try to upgrade
        the User to is_staff before association; 3) if the User does not exist,
        it will try to create a new User which is_staff, before association.
        """
        try:
            user = User.objects.get(username=username, is_staff=True)
        except User.DoesNotExist:
            try:
                user = User.objects.get(username=username)
                user.is_staff = True
                user.save()
            except User.DoesNotExist:
                user = User.objects.create_user(username=username, is_staff=True)
        assert cls.objects.filter(user=user).count() == 0
        p = cls.objects.create(user=user)
        p.save()
        return p


class Experiment(GenericModel):
    """
    Experiment, each object typically consists of multiple Sessions, each
    Session run on a Subject.
    """
    title = models.CharField('experiment title',
                             max_length=200,
                             null=False, blank=False,
                             unique=True)
    description = models.TextField('experiment description',
                                   null=True, blank=True)
    experimenters = models.ManyToManyField(Experimenter,
                                           through='ExperimentRegister',
                                           blank=True)

    def __str__(self):
        return super().__str__() + f' ({self.title})'


class ExperimentRegister(GenericModel):
    """
    Register to associate an Experiment with an Experimenter.

    In NEST, the association between Experiment and Experimenter is
    many-to-many (an Experiment can be associated with more than one
    Experimenters, and an Experimenter can register multiple Experiments),
    and ExperimentRegister is the middle class to establish their
    association.
    """

    experiment = models.ForeignKey(Experiment, on_delete=models.CASCADE)
    experimenter = models.ForeignKey(Experimenter, on_delete=models.CASCADE)


class Session(GenericModel):
    """
    An continous interval within an Experiment, associated with one Subject.

    Multiple Sessions consist of an Experiment. A session must have one
    Subject, and may consists of multiple Rounds.
    """
    experiment: Experiment = models.ForeignKey(Experiment,
                                               on_delete=models.CASCADE)
    subject: Subject = models.ForeignKey(Subject,
                                         on_delete=models.SET_NULL,
                                         null=True, blank=True)

    def __str__(self):
        return super().__str__() + \
               f' ({str(self.experiment)}, {str(self.subject)})'


class Content(GenericModel):
    """
    A.k.a. source. The source material where a Stimulus can be created based on.

    For example, a piece of Content can be a 5-second video clip chosen from
    Stranger Things.
    """

    name = models.CharField('name', max_length=1000)
    content_id = models.IntegerField('content id', null=True, blank=True)
    experiment: Experiment = models.ForeignKey(Experiment,
                                               on_delete=models.CASCADE,
                                               null=True, blank=True)

    def __str__(self):
        return super().__str__() + f' ({self.name}, {self.content_id})'


class Condition(GenericModel):
    """
    A.k.a. HRC, or Hypothetical Reference Circuits.

    In subjective testing lingo, a Condition determines how the Content is
    presented to a Subject. For example, a Condition can be "encoding a video
    at 1000 Kbps, and decoded and displayed it on a screen which is three times
    the screen height from the Subject's eyes".
    """

    name = models.CharField('name', max_length=1000)
    condition_id = models.IntegerField('condition id', null=True, blank=True)
    experiment: Experiment = models.ForeignKey(Experiment,
                                               on_delete=models.CASCADE,
                                               null=True, blank=True)

    def __str__(self):
        return super().__str__() + f' ({self.name})'


class Stimulus(GenericModel):
    """
    A.k.a PVS: Processed Video Sequence.

    A Stimulus is determined by its Content and the Condition it is presented
    in.

    Note that 'path' and 'type' are present in stimulus_config.stimuli but are
    not fields for Stimulus purposely. The reason is that we want path be
    easily changed due to moving the media files around but the db doesn't
    needs to be updated every time.
    """

    content = models.ForeignKey(Content, on_delete=models.SET_NULL, null=True,
                                blank=True)
    condition = models.ForeignKey(Condition, on_delete=models.SET_NULL,
                                  null=True, blank=True)
    stimulus_id = models.IntegerField('stimulus id', null=True, blank=True)
    experiment: Experiment = models.ForeignKey(Experiment,
                                               on_delete=models.CASCADE,
                                               null=True, blank=True)

    def __str__(self):
        return (super().__str__() +
                f' ({self.content}, {self.condition}, {self.stimulus_id})')


class StimulusGroup(GenericModel):
    """
    One StimulusGroup contains multiple StimulusVoteGroups and is evaluated
    in each Round.
    """
    stimulusgroup_id = models.IntegerField('stimulus group id', null=True, blank=True)
    experiment: Experiment = models.ForeignKey(Experiment,
                                               on_delete=models.CASCADE,
                                               null=True, blank=True)

    @property
    def stimuli(self):
        stims = []
        for svg in StimulusVoteGroup.objects.filter(stimulusgroup=self):
            stims += svg.stimuli
        stims = sorted(set(stims), key=lambda x: x.id)
        return stims

    def __str__(self):
        return super().__str__() + f" ({self.stimulusgroup_id} : {','.join([str(stim.id) for stim in self.stimuli])})"


class StimulusVoteGroup(GenericModel):
    """
    A StimulusVoteGroup is a subset of a StimulusGroup. Each Vote is associated
    with a StimulusVoteGroup; each StimulusVoteGroup may have multiple votes.

    The relationship between StimulusVoteGroup and Stimulus is many-to-many:
    - Each Stimulus may belong to more than one StimulusVoteGroup.
    - Each StimulusVoteGroup may involve either one or two Stimuli, depending
    on if it is absolute (e.g. video A has “bad” quality) or relative scale
    (e.g. video A has “imperceptible” distortion compared to B). For the second
     case, the order matters, and this is preserved in the the VoteRegister
     (for example, if video A is order 1 and video B is order 2, then a vote
     of +1 represents that B is +1 better than A).
    """

    stimulus = models.ManyToManyField(Stimulus,
                                      through='VoteRegister',
                                      blank=True)
    stimulusgroup = models.ForeignKey(StimulusGroup,
                                      on_delete=models.CASCADE,
                                      null=True, blank=True)
    stimulusvotegroup_id = models.IntegerField('stimulus vote group id',
                                               null=True, blank=True)
    experiment: Experiment = models.ForeignKey(Experiment,
                                               on_delete=models.CASCADE,
                                               null=True, blank=True)

    @property
    def stimuli(self):
        stims = []
        for vr in VoteRegister.objects.filter(stimulusvotegroup=self):
            stims += [vr.stimulus]
        stims = sorted(set(stims), key=lambda x: x.id)
        return stims

    def __str__(self):
        return super().__str__() + f" ({','.join([str(stim.id) for stim in self.stimuli])})"

    @staticmethod
    def create_stimulusvotegroup_from_stimulus(stimulus: Stimulus,
                                               stimulusgroup: StimulusGroup = None,
                                               stimulusvotegroup_id: int = None,
                                               experiment: Experiment = None) -> StimulusVoteGroup:
        """
        create svg that corresponds to sg and a single stimulus
        """
        d = dict(stimulusgroup=stimulusgroup)
        if stimulusvotegroup_id is not None:
            d.update(dict(stimulusvotegroup_id=stimulusvotegroup_id))
        if experiment is not None:
            d.update(dict(experiment=experiment))

        svg = StimulusVoteGroup(**d)
        svg.save()
        VoteRegister.objects.create(
            stimulusvotegroup=svg,
            stimulus=stimulus,
        )
        return svg

    @staticmethod
    def find_stimulusvotegroups_from_stimulus(stim: Stimulus
                                              ) -> List[StimulusVoteGroup]:
        """
        find svgs that corresponds to single stimulus
        """

        svgs = StimulusVoteGroup.objects.filter(stimulus=stim)
        candidate_svgs = []
        for svg in svgs:
            vrs = VoteRegister.objects.filter(stimulus=stim,
                                              stimulus_order=1,
                                              stimulusvotegroup=svg)
            if vrs.count() > 0:
                assert vrs.count() == 1, \
                    f"expect a unique VoteRegister with svg and stim order 1," \
                    f" but got {vrs.count}: {vrs.all()}."
                candidate_svgs.append(svg)
        return candidate_svgs

    @staticmethod
    def create_stimulusvotegroup_from_stimuli_pair(stimulus1: Stimulus,
                                                   stimulus2: Stimulus,
                                                   stimulusgroup: StimulusGroup = None,
                                                   stimulusvotegroup_id: int = None,
                                                   experiment: Experiment = None) -> StimulusVoteGroup:
        """
        create svg that corresponds to sg and stimuli with order (stim1, stim2)
        """
        d = dict(stimulusgroup=stimulusgroup)
        if stimulusvotegroup_id is not None:
            d.update(dict(stimulusvotegroup_id=stimulusvotegroup_id))
        if experiment is not None:
            d.update(dict(experiment=experiment))
        svg = StimulusVoteGroup(**d)
        svg.save()
        VoteRegister.objects.create(
            stimulusvotegroup=svg,
            stimulus=stimulus1,
            stimulus_order=1,
        )
        VoteRegister.objects.create(
            stimulusvotegroup=svg,
            stimulus=stimulus2,
            stimulus_order=2,
        )
        return svg

    @staticmethod
    def find_stimulusvotegroups_from_stimuli_pair(stim1: Stimulus,
                                                  stim2: Stimulus
                                                  ) -> List[StimulusVoteGroup]:
        """
        find svgs that corresponds to stimuli with order (stim1, stim2)
        """

        # FIXME: the below logic seems convolved, although db_stress_test did
        #  not show much import/export time degradation as the db gets bigger
        svgs = (StimulusVoteGroup.objects.filter(stimulus=stim1)
                & StimulusVoteGroup.objects.filter(stimulus=stim2))
        candidate_svgs = []
        for svg in svgs:
            vrs = VoteRegister.objects.filter(stimulus=stim1,
                                              stimulus_order=1,
                                              stimulusvotegroup=svg)
            if vrs.count() > 0:
                assert vrs.count() == 1, \
                    f"expect a unique VoteRegister with svg and stim1 order 1 and " \
                    f"stim2 order 2, but got {vrs.count}: {vrs.all()}."
                candidate_svgs.append(svg)
        return candidate_svgs


class Round(GenericModel):
    """
    A short interval within a Session. It could be an interval where a Subject
    needs to evaluate the quality of a video Stimulus.
    """
    session: Session = models.ForeignKey(Session, on_delete=models.CASCADE)
    round_id: int = models.IntegerField('round id', null=True, blank=True)
    stimulusgroup: StimulusGroup = models.ForeignKey(StimulusGroup,
                                                     on_delete=models.SET_NULL,
                                                     null=True, blank=True)
    response_sec = models.FloatField('response sec', null=True, blank=True)

    def __str__(self):
        return super().__str__() + " ({}, {}, {})".format(
            self.session,
            self.round_id,
            self.stimulusgroup.stimulusgroup_id if
            self.stimulusgroup is not None else None
        )


class Vote(GenericModel, TypeVersionEnabled):
    """
    Abtract class for a Vote.

    It is associated with a Round of a Experiment Session, and can be
    associated with one (e.g. rate the quality of this video), two (e.g. rate
    the quality of the second video relative to the first video), or multiple
    (e.g. pick the best quality video from a list of videos) Stimuli.
    """
    round = models.ForeignKey(
        Round, on_delete=models.CASCADE,
        # the below are for convenience in tests since
        # don't want to create Round before creating Vote:
        null=True, blank=True,
    )
    stimulusvotegroup = models.ForeignKey(
        StimulusVoteGroup,
        on_delete=models.CASCADE,
        # the below are for convenience in tests since
        # don't want to create StimulusVoteGroup before
        # creating Vote:
        null=True, blank=True,
    )
    score = models.FloatField('score', null=False, blank=False)

    TYPE = 'VOTE'
    VERSION = '1.0'

    def __str__(self):
        return super().__str__() + f' ({self.score})'


class VoteRegister(GenericModel):
    """
    Register to associate a StimulusVoteGroup with a Stimulus.

    In NEST, the association between StimulusVoteGroup and Stimulus is
    many-to-many, and VoteRegister is the middle class to establish their
    association.

    For all the Stimuli associated with a StimulusVoteGroup, the order matters.
    For example, the first Stimulus is the reference and the second is the
    distorted, and the Vote associated with the StimulusVoteGroup is the
    quality of the distorted relative to the reference.
    """

    stimulusvotegroup = models.ForeignKey(StimulusVoteGroup, on_delete=models.CASCADE)
    stimulus = models.ForeignKey(Stimulus, on_delete=models.CASCADE)
    stimulus_order = models.PositiveIntegerField('stimulus order', default=1)


class DiscreteVote(Vote):
    """
    Abstract subclass of Vote, whose vote values come from a discrete set. For
    example, binary vote chosen from {0, 1}.
    """

    __metaclass__ = ABCMeta

    class Meta:
        abstract = True

    @property
    @abstractmethod
    def support(self):
        raise NotImplementedError

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        if 'score' in kwargs:
            assert kwargs['score'] in self.support, \
                f"score {kwargs['score']} is not in the support {self.support}"

    @classmethod
    def assert_vote(cls, vote):
        assert vote in cls.support


class ContinuousVote(Vote):
    """
    Abstract subclass of Vote, whose vote values come from a continuous
    interval. For example, a real-value vote chosen in the interval of [0, 1].
    """

    __metaclass__ = ABCMeta

    class Meta:
        abstract = True

    @property
    @abstractmethod
    def range(self):
        raise NotImplementedError

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        assert len(self.range) == 2 and self.range[0] < self.range[
            1], f"illegal range {self.range}"
        if 'score' in kwargs:
            assert self.range[0] <= kwargs['score'] <= self.range[1], \
                f"score {kwargs['score']} is not in the range {self.range}"

    @classmethod
    def assert_vote(cls, vote):
        assert cls.range[0] <= vote <= cls.range[1]


class FivePointVote(DiscreteVote):
    """
    Discrete Vote from 1, 2, 3, 4, 5.
    """
    support = [1, 2, 3, 4, 5]
    TYPE = 'FIVE_POINT'


class TafcVote(DiscreteVote):
    """
    Two-alternative forced choice (2AFC) Vote from {0, 1}. aka "CcrTwoPointVote".
    """
    support = [0, 1]
    TYPE = '2AFC'


class CcrThreePointVote(DiscreteVote):
    """
    CCR (Comparison Category Rating) Three-point vote from {0, 1, 2}.
    """
    support = [0, 1, 2]
    TYPE = 'CCR_THREE_POINT'


class Zero2HundredVote(ContinuousVote):
    """
    Continuous Vote from the interval [0, 100].
    """
    range = [0, 100]
    TYPE = '0_TO_100'


class ElevenPointVote(DiscreteVote):
    """
    Discrete Vote from 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
    """
    support = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
    TYPE = 'ELEVEN_POINT'


class SevenPointVote(DiscreteVote):
    """
    Discrete Vote from 1, 2, 3, 4, 5, 6, 7
    """
    support = [1, 2, 3, 4, 5, 6, 7]
    TYPE = 'SEVEN_POINT'


class ThreePointVote(DiscreteVote):
    """
    Discrete Vote from 1, 2, 3
    """
    support = [1, 2, 3]
    TYPE = 'THREE_POINT'

nest_site/nest/models.py
Netflix/e2nest
from abc import ABC, ABCMeta, abstractmethod

from nest.helpers import override


class PageWithVideoMixin(object):

    DEFAULT_VIDEO_CONTEXT = {
        'video_show_controls': False,
        'video_display_percentage': 100,
        'preload_videos': False,
    }

    def _assert_video_context(self):
        required_fields = ['video_show_controls', 'video_display_percentage']
        for e in required_fields:
            assert e in self.context, f'parameter {e} is required in context'
        assert isinstance(self.context['video_show_controls'], bool)
        assert 0 < self.context['video_display_percentage'] <= 100
        assert isinstance(self.context['preload_videos'], bool)


class Page(object):

    __metaclass__ = ABCMeta

    DEFAULT_CONTEXT = dict()

    class Meta:
        abstract = True

    def __init__(self, context: dict):
        self.context = dict()

        # get default context
        self.context.update(self.DEFAULT_CONTEXT)
        if isinstance(self, PageWithVideoMixin):
            self.context.update(PageWithVideoMixin.DEFAULT_VIDEO_CONTEXT)

        # get from input context, overriding default
        self.context.update(context)

        # assert
        self._assert_context()
        if isinstance(self, PageWithVideoMixin):
            self._assert_video_context()

    @abstractmethod
    def _assert_context(self):
        raise NotImplementedError

    @property
    @abstractmethod
    def DEFAULT_TEMPLATE(self):
        raise NotImplementedError

    def get_template(self):
        return self.DEFAULT_TEMPLATE

    VERSION = '1.0'


class GenericPage(Page, ABC):
    """Generic page"""
    DEFAULT_TEMPLATE = 'nest/generic.html'

    @override(Page)
    def _assert_context(self):
        required_fields = ['title', 'text_html', 'actions_html']
        for e in required_fields:
            assert e in self.context, f'parameter {e} is required in context'


class AcrPage(Page, PageWithVideoMixin, ABC):  # TODO: generalize AcrPage to support continuous scale, so that there's no need for separate Acr5cPage # noqa E501
    """Absolute Catagory Rating (ACR) page"""
    TEMPLATES = {'interactive': 'nest/acr.html',
                 'standard': 'nest/acr_standard.html'
                 }
    DEFAULT_TEMPLATE = 'interactive'

    DEFAULT_CONTEXT = {
        'instruction_html':
            """<p> Watch a video by clicking on the "Play video" button. The video will loop. <em> Press "space" key or double click the mouse </em> to exit. After that, rate the video by answering the question provided. </p>""",  # noqa E501
        'button': 'Play video',
        'question':
            "On a scale of 1 to 5, how would you rate the video quality?",
        'choices':
            ['5 - Excellent',
             '4 - Good',
             '3 - Fair',
             '2 - Poor',
             '1 - Bad']
    }

    @override(Page)
    def _assert_context(self):
        if 'template_version' not in self.context:
            self.context['template_version'] = self.DEFAULT_TEMPLATE

        if self.context['template_version'] == 'interactive':
            required_fields = ['title', 'instruction_html',
                               'button', 'video',
                               'question', 'choices',
                               'stimulusvotegroup_id']
        elif self.context['template_version'] == 'standard':
            required_fields = ['title', 'instruction_html',
                               'num_plays', 't_gray',
                               'button', 'video',
                               'question', 'choices',
                               'stimulusvotegroup_id']
        else:
            assert False, 'acr methodology only supports interactive and standard template_version'

        for e in required_fields:
            assert e in self.context, f'parameter {e} is required in context'
        if 'session_id' in self.context:
            assert isinstance(self.context['session_id'], int)
        if self.context['template_version'] == 'standard':
            assert self.context['num_plays'] >= 1
            if 'min_num_plays' in self.context:
                assert self.context['num_plays'] >= self.context['min_num_plays'] >= 0
            assert self.context['t_gray'] >= 0
            if 'text_color' in self.context:
                assert len(self.context['text_color']) == 7 and self.context['text_color'].startswith('#'), \
                    'text_color must in the format of #0000FF'
            if 'text_vert_perc' in self.context:
                assert 0 <= self.context['text_vert_perc'] <= 100

    @override(Page)
    def get_template(self):
        return self.TEMPLATES[self.context['template_version']]


class Acr5cPage(Page, PageWithVideoMixin, ABC):
    """Absolute Catagory Rating (ACR) with 5-point continuous scale page"""
    TEMPLATES = {'interactive': 'nest/acr5c.html',
                 'standard': 'nest/acr5c_standard.html'}

    DEFAULT_TEMPLATE = 'interactive'

    DEFAULT_CONTEXT = {
        'instruction_html':
            """<p> Watch a video by clicking on the "Play video" button. The video will loop. <em> Press "space" key or double click the mouse </em> to exit. After that, rate the video by answering the question provided. </p>""",  # noqa E501
        'button': 'Play video',
        'question':
            "On a scale of 0 to 100, how would you rate the video quality?",
        'labels':
            ['Excellent',
             'Good',
             'Fair',
             'Poor',
             'Bad']
    }

    @override(Page)
    def _assert_context(self):
        if 'template_version' not in self.context:
            self.context['template_version'] = self.DEFAULT_TEMPLATE

        if self.context['template_version'] == 'interactive':
            required_fields = ['title', 'instruction_html',
                               'button', 'video',
                               'question', 'labels',
                               'stimulusvotegroup_id']
        elif self.context['template_version'] == 'standard':
            required_fields = ['title', 'instruction_html',
                               'num_plays', 't_gray',
                               'button', 'video',
                               'question', 'labels',
                               'stimulusvotegroup_id']
        else:
            assert False, 'acr5c methodology only supports interactive and standard template_version'

        for e in required_fields:
            assert e in self.context, f'parameter {e} is required in context'
        if 'session_id' in self.context:
            assert isinstance(self.context['session_id'], int)
        if self.context['template_version'] == 'standard':
            assert self.context['num_plays'] >= 1
            if 'min_num_plays' in self.context:
                assert self.context['num_plays'] >= self.context['min_num_plays'] >= 0
            assert self.context['t_gray'] >= 0
            if 'text_color' in self.context:
                assert len(self.context['text_color']) == 7 and self.context['text_color'].startswith('#'), \
                    'text_color must in the format of #0000FF'
            if 'text_vert_perc' in self.context:
                assert 0 <= self.context['text_vert_perc'] <= 100

    @override(Page)
    def get_template(self):
        return self.TEMPLATES[self.context['template_version']]


class DcrPage(Page, PageWithVideoMixin, ABC):
    """Degradation Category Rating (DCR) page"""
    TEMPLATES = {'interactive': 'nest/dcr.html',
                 'standard': 'nest/dcr_standard.html'
                 }
    DEFAULT_TEMPLATE = 'interactive'

    DEFAULT_CONTEXT = {
        'instruction_html':
            """<p> Watch a reference video R, followed by its degraded version D. Each video will loop. <em> Press "space" key or double click the mouse </em> to exit. After that, rate the videos by answering the question provided. </p>""",  # noqa E501
        'button_a': 'Play reference R',
        'button_b': 'Play degraded D',
        'question':
            "Comparing to the reference R, how would you rate the impairment in D?",
        'choices':
            ['5 - Imperceptible',
             '4 - Perceptible, but not annoying',
             '3 - Slightly annoying',
             '2 - Annoying',
             '1 - Very annoying']
    }

    @override(Page)
    def _assert_context(self):
        if 'template_version' not in self.context:
            self.context['template_version'] = self.DEFAULT_TEMPLATE

        if self.context['template_version'] == 'interactive':
            required_fields = ['title', 'instruction_html',
                               'button_a', 'button_b',
                               'video_a', 'video_b',
                               'question', 'choices',
                               'stimulusvotegroup_id']
        elif self.context['template_version'] == 'standard':
            required_fields = ['title', 'instruction_html',
                               'num_plays', 't_gray',
                               'button_a', 'button_b',
                               'video_a', 'video_b',
                               'question', 'choices',
                               'stimulusvotegroup_id']
        else:
            assert False, 'dcr methodology only supports interactive and standard template_version'

        for e in required_fields:
            assert e in self.context, f'parameter {e} is required in context'
        if self.context['template_version'] == 'standard':
            assert self.context['num_plays'] >= 1
            if 'min_num_plays' in self.context:
                assert self.context['num_plays'] >= self.context['min_num_plays'] >= 0
            assert self.context['t_gray'] >= 0
            if 'text_color' in self.context:
                assert len(self.context['text_color']) == 7 and self.context['text_color'].startswith('#'), \
                    'text_color must in the format of #0000FF'
            if 'text_vert_perc' in self.context:
                assert 0 <= self.context['text_vert_perc'] <= 100

    @override(Page)
    def get_template(self):
        return self.TEMPLATES[self.context['template_version']]


class CcrPage(Page, PageWithVideoMixin, ABC):
    """Comparison Category Rating (CCR) page"""
    TEMPLATES = {'interactive': 'nest/ccr.html',
                 'standard': 'nest/ccr_standard.html'
                 }
    DEFAULT_TEMPLATE = 'interactive'

    DEFAULT_CONTEXT = {
        'instruction_html':
            """<p> Watch a Video A, followed by Video B. Each video will loop. <em> Press "space" key or double click the mouse </em> to exit. After that, rate the videos by answering the question provided. </p>""",  # noqa E501
        'button_a': 'Play Video A',
        'button_b': 'Play Video B',
        'question':
            "How do you compare the visual quality of Video A to Video B?",
        'choices':
            ['Video A is better',
             'They are the same',
             'Video B is better'],
    }

    @override(Page)
    def _assert_context(self):
        if 'template_version' not in self.context:
            self.context['template_version'] = self.DEFAULT_TEMPLATE

        if self.context['template_version'] == 'interactive':
            required_fields = ['title', 'instruction_html',
                               'button_a', 'button_b',
                               'video_a', 'video_b',
                               'video_a_to_b_values',
                               'question', 'choices',
                               'stimulusvotegroup_id']
        elif self.context['template_version'] == 'standard':
            required_fields = ['title', 'instruction_html',
                               'num_plays', 't_gray',
                               'button_a', 'button_b',
                               'video_a', 'video_b',
                               'video_a_to_b_values',
                               'question', 'choices',
                               'stimulusvotegroup_id']
        else:
            assert False, f"ccr methodology only supports interactive and standard template_version, but " \
                          f"is: {self.context['template_version']}"

        for e in required_fields:
            assert e in self.context, f'parameter {e} is required in context'

        assert len(self.context['choices']) == len(self.context['video_a_to_b_values']), \
            f"the length of choices and video_a_to_b_values must be equal, but they are: {self.context['choices']} " \
            f"and {self.context['video_a_to_b_values']}"

        if self.context['template_version'] == 'standard':
            assert self.context['num_plays'] >= 1
            if 'min_num_plays' in self.context:
                assert self.context['num_plays'] >= self.context['min_num_plays'] >= 0
            assert self.context['t_gray'] >= 0
            if 'text_color' in self.context:
                assert len(self.context['text_color']) == 7 and self.context['text_color'].startswith('#'), \
                    'text_color must in the format of #0000FF'
            if 'text_vert_perc' in self.context:
                assert 0 <= self.context['text_vert_perc'] <= 100

    @override(Page)
    def get_template(self):
        return self.TEMPLATES[self.context['template_version']]


class SamviqPage(Page, PageWithVideoMixin, ABC):
    """SAMVIQ with continuous quality scale with 5 labels"""
    DEFAULT_TEMPLATE = 'nest/samviq.html'

    DEFAULT_CONTEXT = {
        'instruction_html':
            """<p> Watch a reference video R and rate each of its processed versions. Each video will loop. <em> Press "space" key or double click the mouse </em> to exit. After that, rate the videos by answering the question provided. </p>""",  # noqa E501
        'question':
            "Comparing to the reference R, how would you rate the quality of the videos?",
        'labels':
            ['Excellent',
             'Good',
             'Fair',
             'Poor',
             'Bad'],
    }

    @override(Page)
    def _assert_context(self):
        required_fields = ['title', 'instruction_html',
                           'videos', 'video_ref',
                           'buttons', 'button_ref',
                           'question', 'labels',
                           'stimulusvotegroup_ids']
        for e in required_fields:
            assert e in self.context, f'parameter {e} is required in context'
        assert len(self.context['labels']) == 5
        assert isinstance(self.context['videos'], list)
        assert len(self.context['videos']) > 0
        assert isinstance(self.context['buttons'], list)
        assert len(self.context['buttons']) \
               == len(self.context['videos']) \
               == len(self.context['stimulusvotegroup_ids'])


class Samviq5dPage(Page, PageWithVideoMixin, ABC):
    """SAMVIQ with 5-point discrete quality scale"""
    DEFAULT_TEMPLATE = 'nest/samviq5d.html'

    DEFAULT_CONTEXT = {
        'instruction_html':
            """<p> Watch a reference video R and rate each of its processed versions. Each video will loop. <em> Press "space" key or double click the mouse </em> to exit. After that, rate the videos by answering the question provided. </p>""",  # noqa E501
        'question':
            "Comparing to the reference R, how would you rate the quality of the videos?",
        'choices':
            ['5 - Excellent',
             '4 - Good',
             '3 - Fair',
             '2 - Poor',
             '1 - Bad'],
    }

    @override(Page)
    def _assert_context(self):
        required_fields = ['title', 'instruction_html',
                           'videos', 'video_ref',
                           'buttons', 'button_ref',
                           'question', 'choices',
                           'stimulusvotegroup_ids']
        for e in required_fields:
            assert e in self.context, f'parameter {e} is required in context'
        assert len(self.context['choices']) == 5
        assert isinstance(self.context['videos'], list)
        assert len(self.context['videos']) > 0
        assert isinstance(self.context['buttons'], list)
        assert len(self.context['buttons']) \
               == len(self.context['videos']) \
               == len(self.context['stimulusvotegroup_ids'])


class StatusPage(Page, ABC):
    """
    Status page to list experiment sessions status for a subject.
    Example for tests:
    tests = \
        [
            {'title': 'test 1', 'session': '1', 'status': 'Done', 'action': None},
            {'title': 'test 2', 'session': '5', 'status': 'Done', 'action': None},
            {'title': 'test 2', 'session': '25', 'status': 'Unfinished', 'action': 'Reset'},
            {'title': 'test 3', 'session': '61', 'status': 'New', 'action': 'Start'},
        ]
    """
    DEFAULT_TEMPLATE = 'nest/status.html'

    @override(Page)
    def _assert_context(self):
        required_fields = ['title', 'tests']
        for e in required_fields:
            assert e in self.context, f'parameter {e} is required in context'
        for test in self.context['tests']:
            assert isinstance(test, dict)
            assert 'title' in test
            assert 'session' in test
            assert 'status' in test
            assert 'action' in test
            assert test['status'] in ['Uninitialized', 'Partially initialized',
                                      'New', 'Unfinished', 'In progress', 'Done']
            assert test['action'] in [None, 'Start', 'Reset', 'Continue']
            if test['action'] in ['Reset', 'Start', 'Continue']:
                assert 'action_url' in test


def map_methodology_to_page_class(methodology: str) -> Page.__class__:
    if methodology == 'acr':
        return AcrPage
    elif methodology == 'acr5c':
        return Acr5cPage
    elif methodology == 'dcr':
        return DcrPage
    elif methodology == 'tafc':
        return CcrPage
    elif methodology == 'ccr':
        return CcrPage
    elif methodology == 'samviq5d':
        return Samviq5dPage
    elif methodology == 'samviq':
        return SamviqPage
    else:
        assert False, f'unknown mapping from methodology {methodology} to page class.'


def map_methodology_to_html_id_key(methodology: str) -> str:
    if methodology == 'tafc':
        return 'ccr'
    elif methodology in ['acr', 'acr5c', 'dcr', 'ccr', 'samviq5d', 'samviq']:
        return methodology
    else:
        assert False, f'unknown mapping from methodology {methodology} to HTML id key.'

nest_site/nest/pages.py
Netflix/e2nest
#!/usr/bin/env python3

import argparse

import django
django.setup()

from nest.io import ExperimentUtils  # noqa: E402, I202

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--action", dest="action", nargs=1, type=str,
        help="action to take, options: validate_config, create_experiment, "
             "delete_experiment, add_session, delete_session, "
             "update_first_session_subject, reset_unfinished_first_session",
        required=True)
    parser.add_argument(
        "--config", dest="config_filepath", nargs=1, type=str,
        help="path to config file for creating experiment", required=False)
    parser.add_argument(
        "--experiment", dest="experiment_title", nargs=1, type=str,
        help="experiment title to add session", required=False)
    parser.add_argument(
        "--username", dest="username", nargs=1, type=str,
        help="specify the username of either subject or experimenter; if not "
             "exist, create one. In the case the action is create_experiment, "
             "the username is for the experimenter associated; for all other "
             "action, the username is for the test subject",
        required=False)
    parser.add_argument(
        "--session_id", dest="session_id", nargs=1, type=int,
        help="specify the session ID",
        required=False)
    args = parser.parse_args()
    action = args.action[0]
    config_filepath = args.config_filepath[0] if args.config_filepath else None
    experiment_title = args.experiment_title[0] if args.experiment_title else None
    username = args.username[0] if args.username else None
    session_id = args.session_id[0] if args.session_id else None

    if action == 'validate_config':
        assert config_filepath is not None
        assert experiment_title is None
        assert username is None
        assert session_id is None
        ExperimentUtils.validate_config(
            experiment_config_filepath=config_filepath)
        print(f"config file is valid: {config_filepath}")
    elif action == 'create_experiment':
        assert config_filepath is not None
        assert experiment_title is None, 'experiment_title is specified through config file'
        assert username is not None
        assert session_id is None
        ExperimentUtils.create_experiment(
            experiment_config_filepath=config_filepath,
            experimenter_username=username)
    elif action == 'delete_experiment':
        assert config_filepath is None
        assert experiment_title is not None
        assert username is None
        assert session_id is None
        ExperimentUtils.delete_experiment_by_title(
            experiment_title=experiment_title,
            is_test=False)
    elif action == 'add_session':
        assert config_filepath is None
        assert experiment_title is not None
        assert username is not None
        assert session_id is None
        ExperimentUtils.add_session_to_experiment(
            experiment_title=experiment_title,
            subject_username=username)
    elif action == 'delete_session':
        assert config_filepath is None
        assert experiment_title is not None
        assert username is None
        assert session_id is not None
        ExperimentUtils.delete_session_by_id(
            session_id=session_id,
            experiment_title=experiment_title)
    elif action == 'update_first_session_subject':
        assert config_filepath is None
        assert experiment_title is not None
        assert username is not None
        assert session_id is None
        ec = ExperimentUtils.get_experiment_controller(experiment_title)
        first_session = ec.experiment.session_set.first()
        ExperimentUtils.update_subject_for_session(
            experiment_title=experiment_title,
            session_id=first_session.id,
            subject_username=username)
    elif action == 'reset_unfinished_first_session':
        assert config_filepath is None
        assert experiment_title is not None
        assert username is None
        assert session_id is None
        ec = ExperimentUtils.get_experiment_controller(experiment_title)
        first_session = ec.experiment.session_set.first()
        ExperimentUtils.reset_unfinished_session(
            experiment_title=experiment_title,
            session_id=first_session.id)
    else:
        assert False, f"Unknown action: {action}"

    exit(0)

nest_site/nest/scripts/experiment_tools.py
Netflix/e2nest
#!/usr/bin/env python3

import argparse

import django
from sureal.dataset_reader import DatasetReader

django.setup()

from nest.io import export_sureal_dataset  # noqa: E402, I202, I100

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--experiment", dest="experiment_title", nargs=1, type=str,
        help="title of experiment to export",
        required=True)
    parser.add_argument(
        "--output-dataset", dest="output_dataset_path", nargs=1, type=str,
        help="output sureal-formatted dataset filepath", required=True)

    args = parser.parse_args()
    experiment_title = args.experiment_title[0]
    output_dataset_path = args.output_dataset_path[0]
    dataset = export_sureal_dataset(experiment_title)
    DatasetReader.write_out_dataset(dataset, output_dataset_path)
    exit(0)

nest_site/nest/scripts/export_sureal_dataset.py
Netflix/e2nest
#!/usr/bin/env python3

import argparse

import django
from nest.helpers import import_python_file

django.setup()

from nest.io import import_sureal_dataset  # noqa: E402, I202


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--dataset", dest="dataset_path", nargs=1, type=str,
        help="input sureal-formatted dataset filepath", required=True)
    parser.add_argument(
        "--scale", dest="scale", nargs=1, type=str,
        help="voting scale, currently support: FIVE_POINT, 0_TO_100, 2AFC",
        required=True)
    parser.add_argument(
        "--new-dataset-name", dest="new_dataset_name", nargs=1, type=str,
        help="new dataset name to overwrite the old one. This is useful when "
             "importing the same dataset twice while needing to keep the "
             "dataset_name unique",
        required=False)
    parser.add_argument(
        "--experimenter", dest="experimenter_username", nargs=1, type=str,
        help="username of experimenter; if not exist, create a dummy one",
        required=False)
    parser.add_argument(
        "--write-config", dest="write_config", action='store_true',
        help="write config file to /media/experiment_config directory",
        required=False)

    args = parser.parse_args()
    dataset_path = args.dataset_path[0]
    scale = args.scale[0]
    new_dataset_name = args.new_dataset_name[0] \
        if args.new_dataset_name is not None else None
    experimenter_username = args.experimenter_username[0] \
        if args.experimenter_username is not None else None
    write_config = args.write_config
    dataset = import_python_file(dataset_path)
    if new_dataset_name is not None:
        dataset.dataset_name = new_dataset_name
    import_sureal_dataset(dataset, scale,
                          experimenter_username=experimenter_username,
                          write_config=write_config)
    exit(0)

nest_site/nest/scripts/import_sureal_dataset.py
Netflix/e2nest
#!/usr/bin/env python3

import argparse

import django

django.setup()

from nest.io import UserUtils  # noqa: E402, I202


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--usernames", dest="usernames", nargs=1, type=str,
        help="list of usernames, separated by comma "
             "(e.g. zli@catflix.com,lkrasula@catflix.com)",
        required=True)
    parser.add_argument(
        "--action", dest="action", nargs=1, type=str,
        help="action to take, options: create_user, set_password",
        required=True)
    parser.add_argument(
        "--is_staff", dest="is_staff", action='store_true',
        help="set user as is_staff")
    parser.add_argument(
        "--is_superuser", dest="is_superuser", action='store_true',
        help="set user as is_superuser")
    parser.add_argument(
        "--password", dest="password", nargs=1, type=str,
        help="password")
    args = parser.parse_args()
    usernames = args.usernames[0].split(',')
    action = args.action[0]
    is_staff = args.is_staff
    is_superuser = args.is_superuser
    password = args.password[0] if args.password else None

    if action == 'create_user':
        assert password is None
        for username in usernames:
            UserUtils.create_user(username, is_staff, is_superuser)
    elif action == 'set_password':
        assert is_staff is False
        assert is_superuser is False
        assert password is not None
        for username in usernames:
            UserUtils.set_password(username, password)
    else:
        assert False, f"Unknown action: {action}"

    exit(0)

nest_site/nest/scripts/user_tools.py
Netflix/e2nest
import json
import logging
import os
import random
import re
import string
import tempfile
from functools import update_wrapper
from time import time
from typing import Union

from django.apps import apps
from django.contrib.admin import AdminSite
from django.contrib.auth import REDIRECT_FIELD_NAME
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.template.response import TemplateResponse
from django.urls import path, re_path, reverse, reverse_lazy
from django.utils.decorators import method_decorator
from django.utils.functional import LazyObject
from django.utils.module_loading import import_string
from django.utils.translation import gettext_lazy
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_protect
from django.views.generic import RedirectView
from nest_site.settings import MEDIA_URL
from sureal.dataset_reader import DatasetReader

from .config import ExperimentConfig, NestConfig, StimulusConfig
from .helpers import override
from .pages import Acr5cPage, AcrPage, CcrPage, DcrPage, GenericPage, map_methodology_to_html_id_key, \
    map_methodology_to_page_class, Samviq5dPage, SamviqPage, StatusPage
logging.basicConfig()
logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0])
logger.setLevel('INFO')


class ExperimentMixin(object):

    @staticmethod
    def _is_test_environment(request):
        if 'SERVER_NAME' in request.environ and \
                request.environ['SERVER_NAME'] == 'testserver':
            is_test_env = True
        else:
            is_test_env = False
        return is_test_env

    @staticmethod
    def get_experiment_config_filepath(experiment_title, is_test):
        if is_test:
            config_filepath = NestConfig.tests_workdir_path(
                'media', 'experiment_config', f"{experiment_title}.json")
        else:
            config_filepath = NestConfig.media_path(
                'experiment_config', f"{experiment_title}.json")
        return config_filepath

    @classmethod
    def _load_experiment_config(cls, experiment_title: str, request):
        is_test = cls._is_test_environment(request)
        config = cls._load_experiment_config2(experiment_title, is_test)
        return config

    @classmethod
    def _load_experiment_config2(cls, experiment_title, is_test):
        config_filepath = cls.get_experiment_config_filepath(experiment_title,
                                                             is_test)
        with open(config_filepath, "rt") as fp:
            config = json.load(fp)
        return config

    def _get_experiment_controller(self, exp, request):
        config = self._load_experiment_config(exp.title, request)
        ec = self._get_experiment_controller2(exp, config)
        return ec

    @staticmethod
    def _get_experiment_controller2(exp, config):
        from .control import ExperimentController
        scfg = StimulusConfig(config['stimulus_config'])
        ecfg = ExperimentConfig(stimulus_config=scfg,
                                config=config['experiment_config'])
        ec = ExperimentController(experiment=exp, experiment_config=ecfg)
        return ec


class NestAdminSite(AdminSite, ExperimentMixin):
    site_title = gettext_lazy('e2nest admin')
    site_header = gettext_lazy('e2nest: Media Subjective Testing Admin')
    index_title = gettext_lazy('Site administration')

    @override(AdminSite)
    def get_urls(self):

        def wrap(view, cacheable=False):
            def wrapper(*args, **kwargs):
                return self.admin_view(view, cacheable)(*args, **kwargs)
            wrapper.admin_site = self
            return update_wrapper(wrapper, view)

        urlpatterns = list()
        urlpatterns += [path('nestexp', wrap(self.nestexp), name='nestexp')]
        urlpatterns += [re_path(r'^nestexp/download_sureal/(?P<experiment_id>[0-9]+)$',
                                wrap(self.download_sureal), name='download_sureal')]
        urlpatterns += [re_path(r'^nestexp/download_sureal_alt/(?P<experiment_id>[0-9]+)$',
                                wrap(self.download_sureal_alt), name='download_sureal_alt')]
        urlpatterns += [re_path(r'^nestexp/download_nest/(?P<experiment_id>[0-9]+)$',
                                wrap(self.download_nest), name='download_nest')]
        urlpatterns += [re_path(r'^nestexp/download_nest_csv/(?P<experiment_id>[0-9]+)$',
                                wrap(self.download_nest_csv), name='download_nest_csv')]
        urlpatterns += super().get_urls()
        return urlpatterns

    @method_decorator(never_cache)
    def nestexp(self, request, extra_context=None):
        from .models import Experiment
        experiments = Experiment.objects.order_by('-id')
        context = {
            **self.each_context(request),
            'title': 'NEST experiments',
            'experiments': experiments,
            **(extra_context or {}),
        }
        request.current_app = self.name
        return TemplateResponse(request, self.index_template or 'admin/nestexp.html', context)

    @method_decorator(never_cache)
    def download_sureal(self, request, experiment_id):
        from .io import export_sureal_dataset
        from .models import Experiment
        experiment = Experiment.objects.get(id=experiment_id)
        dataset = export_sureal_dataset(experiment.title)
        with tempfile.NamedTemporaryFile(mode='r+t') as tf:
            DatasetReader.write_out_dataset(dataset, tf.name)
            if os.path.exists(tf.name):
                with open(tf.name, 'rb') as fh:
                    response = HttpResponse(fh.read(), content_type="text/plain")
                    response['Content-Disposition'] = f'attachment; filename={experiment.title}.vote.py'
                    return response
            raise Http404

    @method_decorator(never_cache)
    def download_sureal_alt(self, request, experiment_id):
        from .io import export_sureal_dataset
        from .models import Experiment
        experiment = Experiment.objects.get(id=experiment_id)
        dataset = export_sureal_dataset(experiment.title, ignore_against=True)
        with tempfile.NamedTemporaryFile(mode='r+t') as tf:
            DatasetReader.write_out_dataset(dataset, tf.name)
            if os.path.exists(tf.name):
                with open(tf.name, 'rb') as fh:
                    response = HttpResponse(fh.read(), content_type="text/plain")
                    response['Content-Disposition'] = f'attachment; filename={experiment.title}.vote.alt.py'
                    return response
            raise Http404

    @method_decorator(never_cache)
    def download_nest(self, request, experiment_id):
        from .models import Experiment
        experiment = Experiment.objects.get(id=experiment_id)
        ec = self._get_experiment_controller(experiment, request)
        einfo: dict = ec.get_experiment_info()
        with tempfile.NamedTemporaryFile(mode='r+t') as tf:
            with open(tf.name, 'wt') as f:
                json.dump(einfo, f, indent=2)
            if os.path.exists(tf.name):
                with open(tf.name, 'rb') as fh:
                    response = HttpResponse(fh.read(), content_type="application/json")
                    response['Content-Disposition'] = f'attachment; filename={experiment.title}.vote.json'
                    return response
            raise Http404

    @method_decorator(never_cache)
    def download_nest_csv(self, request, experiment_id):
        import pandas as pd
        from .control import ExperimentController
        from .models import Experiment
        experiment = Experiment.objects.get(id=experiment_id)
        ec = self._get_experiment_controller(experiment, request)
        einfo: dict = ec.get_experiment_info()
        with tempfile.NamedTemporaryFile(mode='r+t') as tf:
            df: pd.DataFrame = ExperimentController.denormalize_experiment_info(einfo)
            df.to_csv(tf.name, index=False)
            if os.path.exists(tf.name):
                with open(tf.name, 'rb') as fh:
                    response = HttpResponse(fh.read(), content_type="text/plain")
                    response['Content-Disposition'] = f'attachment; filename={experiment.title}.vote.csv'
                    return response
            raise Http404


class NestSite(ExperimentMixin):
    """
    An NestSite object encapsulates an instance of the Django nest application,
    ready to be hooked in to your URLconf.
    """

    # Text to put at the end of each page's <title>.
    site_title = gettext_lazy('e2nest')

    # Text to put in each page's <h1>.
    site_header = gettext_lazy('e2nest: Media Subjective Testing Platform')

    # Text to put at the top of the welcome page.
    index_title = gettext_lazy('e2nest')

    # URL for the "View site" link at the top of each page.
    site_url = '/'

    login_form = None
    login_template = None
    logout_template = None
    password_change_template = None
    password_change_done_template = None
    index_template = None

    def __init__(self, name='nest'):
        self.name = name

    @property
    def urls(self):
        return self.get_urls(), 'nest', self.name

    def has_permission(self, request):
        """
        Return True if the given HttpRequest has permission to view
        *at least one* page in the site.
        """
        return request.user.is_active

    def user_view(self, view, cacheable=False):
        """
        Decorator to create an user view attached to this site. This
        wraps the view and provides permission checking by calling
        ``self.has_permission``.

        By default, user_views are marked non-cacheable using the
        ``never_cache`` decorator. If the view can be safely cached, set
        cacheable=True.
        """
        def inner(request, *args, **kwargs):
            if not self.has_permission(request):
                if request.path == reverse('nest:logout', current_app=self.name):
                    index_path = reverse('nest:index', current_app=self.name)
                    return HttpResponseRedirect(index_path)
                # Inner import to prevent django.contrib.admin (app) from
                # importing django.contrib.auth.models.User (unrelated model).
                from django.contrib.auth.views import redirect_to_login
                return redirect_to_login(
                    request.get_full_path(),
                    reverse('nest:login', current_app=self.name)
                )
            return view(request, *args, **kwargs)
        if not cacheable:
            inner = never_cache(inner)
        # We add csrf_protect here so this function can be used as a utility
        # function for any view, without having to repeat 'csrf_protect'.
        if not getattr(view, 'csrf_exempt', False):
            inner = csrf_protect(inner)
        return update_wrapper(inner, view)

    def get_urls(self):

        def wrap(view, cacheable=False):
            def wrapper(*args, **kwargs):
                return self.user_view(view, cacheable)(*args, **kwargs)
            wrapper.admin_site = self
            return update_wrapper(wrapper, view)

        urlpatterns = [
            path('login/', self.login, name='login'),
            path('logout/', wrap(self.logout), name='logout'),
            path('password_change/',  wrap(self.password_change, cacheable=True), name='password_change'),
            path('password_change/done/', wrap(self.password_change_done, cacheable=True), name='password_change_done'),
            path('', RedirectView.as_view(url=reverse_lazy('nest:status'), permanent=False), name='index'),
            path('status/', wrap(self.status), name='status'),
            path('cookie/', wrap(self.cookie), name='cookie'),
            re_path(r'^session/(?P<session_id>\d+)/reset/$', wrap(self.reset_session), name='reset_session'),
            re_path(r'^session/(?P<session_id>\d+)/start/$', wrap(self.start_session), name='start_session'),
            re_path(r'^session/(?P<session_id>\d+)/step/$', wrap(self.step_session), name='step_session'),

            # temp:
            path('instruction_demo/', self.instruction_demo, name='instruction_demo'),
            path('acr_demo/', self.acr_demo, name='acr_demo'),
            path('acr5c_demo/', self.acr5c_demo, name='acr5c_demo'),
            path('acr5c_standard_demo/', self.acr5c_standard_demo, name='acr5c_standard_demo'),
            path('dcr_demo/', self.dcr_demo, name='dcr_demo'),
            path('dcr_standard_demo/', self.dcr_standard_demo, name='dcr_standard_demo'),
            path('hdr_demo_meridian/', self.hdr_demo_meridian, name='hdr_demo_meridian'),
            path('hdr_demo_sollevante/', self.hdr_demo_sollevante, name='hdr_demo_sollevante'),
            path('hdr_demo_sparks/', self.hdr_demo_sparks, name='hdr_demo_sparks'),
            path('tafc_demo/', self.tafc_demo, name='tafc_demo'),
            path('tafc_standard_demo/', self.tafc_standard_demo, name='tafc_standard_demo'),
            path('ccr_demo/', self.ccr_demo, name='ccr_demo'),
            path('ccr_standard_demo/', self.ccr_standard_demo, name='ccr_standard_demo'),
            path('dcr7d_standard_demo/', self.dcr7d_standard_demo, name='dcr7d_standard_demo'),
            path('dcr11d_demo/', self.dcr11d_demo, name='dcr11d_demo'),
            path('dcr11d_standard_demo/', self.dcr11d_standard_demo, name='dcr11d_standard_demo'),
            path('samviq_demo/', self.samviq_demo, name='samviq_demo'),
            path('samviq5d_demo/', self.samviq5d_demo, name='samviq5d_demo'),
            path('aom_demo_with_preload_s3/', self.aom_demo_with_preload_s3, name='aom_demo_with_preload_s3'),
            path('demo_4kplus_with_preload_s3/', self.demo_4kplus_with_preload_s3, name='demo_4kplus_with_preload_s3'),
            path('demo_4kplus_with_preload_s3_b/', self.demo_4kplus_with_preload_s3_b, name='demo_4kplus_with_preload_s3_b'),
        ]
        return urlpatterns

    def each_context(self, request):
        """
        Return a dictionary of variables to put in the template context for
        *every* page in the site.

        For sites running on a subpath, use the SCRIPT_NAME value if site_url
        hasn't been customized.
        """
        script_name = request.META['SCRIPT_NAME']
        site_url = script_name if self.site_url == '/' and script_name else self.site_url
        return {
            'site_title': self.site_title,
            'site_header': self.site_header,
            'site_url': site_url,
            'has_permission': self.has_permission(request),
            'is_popup': False,
        }

    @method_decorator(never_cache)
    def login(self, request, extra_context=None):
        """
        Display the login form for the given HttpRequest.
        """
        if request.method == 'GET' and self.has_permission(request):
            # Already logged-in, redirect to welcome
            index_path = reverse('nest:index', current_app=self.name)
            return HttpResponseRedirect(index_path)

        from django.contrib.auth.views import LoginView
        # Since this module gets imported in the application's root package,
        # it cannot import models from other applications at the module level,
        # and django.contrib.admin.forms eventually imports User.
        from django.contrib.admin.forms import AuthenticationForm
        context = {
            **self.each_context(request),
            'title': 'Log in',
            'app_path': request.get_full_path(),
            'username': request.user.get_username(),
        }
        if (REDIRECT_FIELD_NAME not in request.GET and
                REDIRECT_FIELD_NAME not in request.POST):
            context[REDIRECT_FIELD_NAME] = reverse('nest:index', current_app=self.name)
        context.update(extra_context or {})

        defaults = {
            'extra_context': context,
            'authentication_form': self.login_form or AuthenticationForm,
            'template_name': self.login_template or 'admin/login.html',
        }
        request.current_app = self.name
        return LoginView.as_view(**defaults)(request)

    @method_decorator(never_cache)
    def logout(self, request, extra_context=None):
        """
        Log out the user for the given HttpRequest.

        This should *not* assume the user is already logged in.
        """
        defaults = {
            'extra_context': {
                **self.each_context(request),
                # Since the user isn't logged out at this point, the value of
                # has_permission must be overridden.
                'has_permission': False,
                **(extra_context or {})
            },
        }
        if self.logout_template is not None:
            defaults['template_name'] = self.logout_template
        request.current_app = self.name
        from .views import NestLogoutView
        response = NestLogoutView.as_view(**defaults)(request)

        # clear cookie when logging out
        self._clear_test_cookie(request)
        self._clear_session_cookie(request)

        return response

    @staticmethod
    def _set_test_cookie(request):
        request.session.set_test_cookie()

    @staticmethod
    def _test_cookie_worked(request):
        return request.session.test_cookie_worked()

    @staticmethod
    def _clear_test_cookie(request):
        if request.session.test_cookie_worked():
            request.session.delete_test_cookie()

    @staticmethod
    def _get_session_steps_from_cookie(request):
        return request.session.setdefault('steps', [])

    @staticmethod
    def _get_session_id_from_cookie(request):
        return request.session.setdefault('session_id', None)

    @staticmethod
    def _set_session_cookie(request, val):
        assert isinstance(val, dict) and 'steps' in val and 'session_id' in val
        request.session['steps'] = val['steps']
        request.session['session_id'] = val['session_id']

    @staticmethod
    def _get_round_response_sec_from_cookie(request, round_ID: int) -> Union[float, None]:
        return request.session.setdefault('round_response_sec', dict()).setdefault(str(round_ID), None)

    @staticmethod
    def _set_round_response_sec_in_cookie(request, round_ID: int, round_response_sec: float):
        request.session.setdefault('round_response_sec', dict())[str(round_ID)] = round_response_sec
        # since modification is not directly on session, but on dict of dict,
        # need to explicitly tell the session has been modified:
        request.session.modified = True

    @staticmethod
    def _clear_session_cookie(request):
        if 'session_id' in request.session:
            del request.session['session_id']
        if 'steps' in request.session:
            del request.session['steps']
        if 'round_response_sec' in request.session:
            del request.session['round_response_sec']

    def password_change(self, request, extra_context=None):
        """
        Handle the "change password" task -- both form display and validation.
        """
        from django.contrib.admin.forms import AdminPasswordChangeForm
        from .views import NestPasswordChangeView
        url = reverse('nest:password_change_done', current_app=self.name)
        defaults = {
            'form_class': AdminPasswordChangeForm,
            'success_url': url,
            'extra_context': {**self.each_context(request), **(extra_context or {})},
        }
        if self.password_change_template is not None:
            defaults['template_name'] = self.password_change_template
        request.current_app = self.name
        return NestPasswordChangeView.as_view(**defaults)(request)

    def password_change_done(self, request, extra_context=None):
        """
        Display the "success" page after a password change.
        """
        from .views import NestPasswordChangeDoneView
        defaults = {
            'extra_context': {**self.each_context(request), **(extra_context or {})},
        }
        if self.password_change_done_template is not None:
            defaults['template_name'] = self.password_change_done_template
        request.current_app = self.name
        return NestPasswordChangeDoneView.as_view(**defaults)(request)

    @method_decorator(never_cache)
    def status(self, request, extra_context=None):
        from .control import SessionStatus
        from .models import Experiment, Session, Subject

        username: str = request.user.get_username()

        subj: Subject = Subject.find_by_username(username)
        tests = []
        for session in Session.objects.filter(subject=subj):
            exp: Experiment = session.experiment
            ec = self._get_experiment_controller(exp, request)
            ss = ec.get_session_status(session)
            extra = {}
            if ss == SessionStatus.UNINITIALIZED:
                status = 'Uninitialized'
                action = None
            elif ss == SessionStatus.PARTIALLY_INITIALIZED:
                status = 'Partially initialized'
                action = None
            elif ss == SessionStatus.INITIALIZED:
                session_id_from_cookie = self._get_session_id_from_cookie(request)
                steps_performed = self._get_session_steps_from_cookie(request)
                if session_id_from_cookie == session.id and len(steps_performed) > 0:
                    # This would allow a session to be cached and resumed, even
                    # after a user goes back to the main page. The session will
                    # only gets cleaned if 1) user logs out, or 2) user starts
                    # another session.
                    status = 'In progress'
                    action = 'Continue'
                    extra['action_url'] = \
                        reverse('nest:step_session', kwargs={"session_id": session.id})
                else:
                    status = 'New'
                    action = 'Start'
                    extra['action_url'] = \
                        reverse('nest:start_session', kwargs={"session_id": session.id})
            elif ss == SessionStatus.PARTIALLY_FINISHED:
                status = 'Unfinished'
                action = 'Reset'
                extra['action_url'] = \
                    reverse('nest:reset_session', kwargs={"session_id": session.id})
            elif ss == SessionStatus.FINISHED:
                status = 'Done'
                action = None
            else:
                assert False, F"unknown status: {ss}"
            tests.append({
                'title': exp.title,
                'session': session.id,
                'status': status,
                'action': action,
                **extra,
            })

        page = StatusPage({
            'title': f"Welcome, {username}!",
            'tests': tests
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name

        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def cookie(self, request, extra_context=None):
        session_id_from_cookie = self._get_session_id_from_cookie(request)
        steps_performed = self._get_session_steps_from_cookie(request)
        title = 'Cookie status'
        proceed_url = reverse('nest:status')
        cookie_json = json.dumps({
            'session_id': session_id_from_cookie,
            'steps_performed': steps_performed
        }, indent=4)
        text_html = f"""<p> {cookie_json} </p>"""
        actions_html = f""" <p> <a class="button" href="{proceed_url}"  id="start">Main page</a> </p>"""
        page = GenericPage({'title': title, 'text_html': text_html, 'actions_html': actions_html})
        context = {**self.each_context(request), **page.context}
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def instruction_demo(self, request, extra_context=None):
        title = 'Instruction'

        text_html = ""
        text_html += """ <p> In this session, you will be asked to evaluate the annoyance caused by presence of banding artifact in a set of videos. </p> """  # noqa E501
        text_html += """ <p> If you don't know what banding artifacts are, it would be good idea to contact the author <b>ptandon@catflix.com</b> before proceeding with the test. </p> """  # noqa E501
        text_html += """ <p> There are two phases, a training and a testing phase. In the training, you will see 6 videos and will be asked to rate the presence of banding artifacts. The videos are chosen to span the scale and there is an expected score for each of the video to provide an anchor for the testing videos. </p> """  # noqa E501
        text_html += """ <p> Next, the testing phase will start and at this point, your scores will be recorded.</b> </p> """  # noqa E501
        text_html += """ <p> To play a video, click on the <b>"Play Video"</b> button. </p> """  # noqa E501
        text_html += """ <p> <b style='color:red !important;'>Please, maintain the viewing distance of approximately 1.5 times the monitor height.</b> </p> """  # noqa E501
        text_html += """ <p> Video clips will loop over, <b>press "space" key or double click mouse </b> to terminate the video. A slider will appear and you will be able to rate annoyance caused by the presence of banding in the video on the 0-100 scale, qualitatively from unwatchable to imperceptible. Please try to focus only on the banding artifacts. For all clips rate the worst case banding seen during video playback, even for videos with multiple scenes. </p>  """  # noqa E501
        text_html += """ <p> Once you have scored the video, click the "Next" button to fetch next video and repeat. </p> """  # noqa E501
        text_html += """ <p> Progress is shown for training and testing cases on top, near instructions. </p> """  # noqa E501

        actions_html = \
            f"""
            <p>
                <a class="button" href="{reverse_lazy('nest:instruction_demo')}"  id="start">Start Evaluation</a>
            </p>
            """
        page = GenericPage({
            'title': title,
            'text_html': text_html,
            'actions_html': actions_html,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def acr_demo(self, request, extra_context=None):
        page = AcrPage({
            'title': "Round 1 of 10",
            'video': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_display_percentage': 75,
            'video_show_controls': True,
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def acr5c_demo(self, request, extra_context=None):
        page = Acr5cPage({
            'title': "Round 1 of 10",
            'video': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"), # noqa E501
            'video_display_percentage': 75,
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def acr5c_standard_demo(self, request, extra_context=None):
        page = Acr5cPage({
            'title': "Round 1 of 10",
            'instruction_html': """ <p> Rate the video by answering the question below. </p>""",
            'video': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"), # noqa E501
            'template_version': 'standard',
            't_gray': 0,
            'num_plays': 2, 'min_num_plays': 0,
            'button': 'Watch the video',
            'video_show_controls': False,
            'video_display_percentage': 75,
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def dcr_demo(self, request, extra_context=None):
        page = DcrPage({
            'title': 'Round 1 of 10',
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'video_display_percentage': 75,
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def dcr_standard_demo(self, request, extra_context=None):
        page = DcrPage({
            'title': 'Round 1 of 10',
            'instruction_html': """ <p> Rate the videos by answering the question below. </p>""",
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'template_version': "standard",
            'num_plays': 2, 'min_num_plays': 0,
            't_gray': 1000,
            'video_show_controls': True,
            'video_display_percentage': 75,
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def hdr_demo_meridian(self, request, extra_context=None):
        page = SamviqPage({
            'title': 'Round 1 of 10',
            'video_ref': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"), # noqa E501
            'button_ref': 'Reference',
            'videos': [os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__Dovi5Vvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf107.81_phonevmaf106.05_psnr24.30_kbps6441.59.m3u8"), # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__Hdr10Vvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf93.88_phonevmaf101.62_psnr42.45_kbps6509.97.mp4"), # noqa E501
                       ],
            'buttons': ['A', 'B'],
            'stimulusvotegroup_ids': [0, 1],
            'video_display_percentage': 100,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def hdr_demo_sollevante(self, request, extra_context=None):
        page = SamviqPage({
            'title': 'Round 1 of 10',
            'video_ref': os.path.join(MEDIA_URL, "mp4/samples/SolLevante/SolLevante10k_B__0_7_0_12__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf85.63_phonevmaf95.85_psnr26.99_kbps7275.37.mp4"), # noqa E501
            'button_ref': 'Reference',
            'videos': [os.path.join(MEDIA_URL, "mp4/samples/SolLevante/SolLevante10k_B__0_7_0_12__Dovi5Vvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf97.35_phonevmaf102.00_psnr18.69_kbps6954.28.m3u8"), # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/SolLevante/SolLevante10k_B__0_7_0_12__Hdr10Vvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf84.91_phonevmaf97.04_psnr37.76_kbps6894.95.mp4"), # noqa E501
                       ],
            'buttons': ['A', 'B'],
            'stimulusvotegroup_ids': [0, 1],
            'video_display_percentage': 100,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def hdr_demo_sparks(self, request, extra_context=None):
        page = SamviqPage({
            'title': 'Round 1 of 10',
            'video_ref': os.path.join(MEDIA_URL, "mp4/samples/Sparks/Sparks_A__2_0_2_5__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf84.08_phonevmaf96.96_psnr35.37_kbps6075.91.mp4"), # noqa E501
            'button_ref': 'Reference',
            'videos': [os.path.join(MEDIA_URL, "mp4/samples/Sparks/Sparks_A__2_0_2_5__Dovi5Vvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf96.18_phonevmaf102.44_psnr22.73_kbps6206.91.m3u8"), # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Sparks/Sparks_A__2_0_2_5__Hdr10Vvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf83.87_phonevmaf96.84_psnr37.11_kbps6081.28.mp4"), # noqa E501
                       ],
            'buttons': ['A', 'B'],
            'stimulusvotegroup_ids': [0, 1],
            'video_display_percentage': 100,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def tafc_demo(self, request, extra_context=None):
        from .models import TafcVote
        page = CcrPage({
            'title': 'Round 1 of 10',
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'video_a_to_b_values': TafcVote.support,
            'question': 'Between Video A and Video B, which video has the better visual quality?',
            'choices':
                ['Video A is better',
                 'Video B is better'],
            'video_display_percentage': 75,
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def tafc_standard_demo(self, request, extra_context=None):
        from .models import TafcVote
        page = CcrPage({
            'title': 'Round 1 of 10',
            'instruction_html': "",
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'video_a_to_b_values': TafcVote.support,
            'question': 'Between Video A and Video B, which video has the better visual quality?',
            'choices':
                ['Video A is better',
                 'Video B is better'],
            'template_version': 'standard',
            'num_plays': 2, 'min_num_plays': 0,
            't_gray': 1000, 'text_color': '#FFFFFF',
            'text_vert_perc': 45,  # text vertical position percentage
            'video_display_percentage': 75,
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def ccr_demo(self, request, extra_context=None):
        from .models import CcrThreePointVote
        page = CcrPage({
            'title': 'Round 1 of 10',
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'video_a_to_b_values': CcrThreePointVote.support,
            'video_display_percentage': 75,
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def ccr_standard_demo(self, request, extra_context=None):
        from .models import CcrThreePointVote
        page = CcrPage({
            'title': 'Round 1 of 10',
            'instruction_html': "",
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'video_a_to_b_values': CcrThreePointVote.support,
            'template_version': 'standard',
            'num_plays': 2, 'min_num_plays': 0,
            't_gray': 1000, 'text_color': '#FFFFFF',
            'text_vert_perc': 45,  # text vertical position percentage
            'video_display_percentage': 100,
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def dcr11d_demo(self, request, extra_context=None):
        page = DcrPage({
            'title': 'Round 1 of 10',
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'video_display_percentage': 75,
            'choices':
                ['11 - Imperceptible',
                 '10 - Slightly perceptible somewhere',
                 '9 - Slightly perceptible everywhere',
                 '8 - Perceptible somewhere',
                 '7 - Perceptible everywhere',
                 '6 - Clearly perceptible somewhere',
                 '5 - Clearly perceptible everywhere',
                 '4 - Annoying somewhere',
                 '3 - Annoying everywhere',
                 '2 - Severely annoying somewhere',
                 '1 - Severely annoying everywhere'],
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def dcr7d_standard_demo(self, request, extra_context=None):
        page = DcrPage({
            'title': 'Round 1 of 10',
            'instruction_html': """ <p> Rate the videos by answering the question below. </p>""",
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'template_version': "standard",
            'num_plays': 2, 'min_num_plays': 0,
            't_gray': 1000,
            'video_show_controls': True,
            'video_display_percentage': 75,
            'choices':
                ['7 - Imperceptible',
                 '6 - Slightly perceptible',
                 '5 - Perceptible',
                 '4 - Clearly perceptible',
                 '3 - Annoying',
                 '2 - Severely annoying',
                 '1 - Unwatchable'],
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def dcr11d_standard_demo(self, request, extra_context=None):
        page = DcrPage({
            'title': 'Round 1 of 10',
            'instruction_html': """ <p> Rate the videos by answering the question below. </p>""",
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'template_version': "standard",
            'num_plays': 2, 'min_num_plays': 0,
            't_gray': 1000,
            'video_show_controls': True,
            'video_display_percentage': 75,
            'choices':
                ['11 - Imperceptible',
                 '10 - Slightly perceptible somewhere',
                 '9 - Slightly perceptible everywhere',
                 '8 - Perceptible somewhere',
                 '7 - Perceptible everywhere',
                 '6 - Clearly perceptible somewhere',
                 '5 - Clearly perceptible everywhere',
                 '4 - Annoying somewhere',
                 '3 - Annoying everywhere',
                 '2 - Severely annoying somewhere',
                 '1 - Severely annoying everywhere'],
            'stimulusvotegroup_id': 0,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def samviq5d_demo(self, request, extra_context=None):
        page = Samviq5dPage({
            'title': 'Round 1 of 10',
            'video_ref': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'button_ref': 'Reference',
            'videos': [os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       ],
            'buttons': ['A', 'B', 'C', 'D'],
            'stimulusvotegroup_ids': [0, 1, 2, 3],
            'video_display_percentage': 75,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def samviq_demo(self, request, extra_context=None):
        page = SamviqPage({
            'title': 'Round 1 of 10',
            'video_ref': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"), # noqa E501
            'button_ref': 'Reference',
            'videos': [os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"), # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"), # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"), # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"), # noqa E501
                       ],
            'buttons': ['A', 'B', 'C', 'D'],
            'stimulusvotegroup_ids': [0, 1, 2, 3],
            'video_display_percentage': 75,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def aom_demo_with_preload_s3(self, request, extra_context=None):
        page = SamviqPage({
            'title': 'Round 1 of 10',
            'video_ref': 'https://netflix-encoding-subjective-testing.s3.us-west-1.amazonaws.com/media/mp4/aom_demo_stimuli/DrivingPOV3.5.A.mp4',  # noqa E501
            'button_ref': 'Reference',
            'videos': ['https://netflix-encoding-subjective-testing.s3.us-west-1.amazonaws.com/media/mp4/aom_demo_stimuli/Marathon2.3.A.mp4',  # noqa E501
                       'https://netflix-encoding-subjective-testing.s3.us-west-1.amazonaws.com/media/mp4/aom_demo_stimuli/TallBuildings2.0.A.mp4',  # noqa E501
                       ],
            'buttons': ['A', 'B'],
            'stimulusvotegroup_ids': [0, 1],
            'video_display_percentage': 100,
            'preload_videos': True,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def demo_4kplus_with_preload_s3(self, request, extra_context=None):
        page = SamviqPage({
            'title': 'Round 1 of 10',
            'video_ref': 'https://netflix-encoding-subjective-testing.s3.us-west-1.amazonaws.com/media/mp4/4kplus_study_demo/LifeInColor_B/LifeInColor_B__3_22_3_27__SdrVvhevceCrfVE__3820_2160__qp14_enable_audio_False_level_51_vmaf63.88_phonevmaf83.20_psnr23.51_kbps39289.47.mp4',  # noqa E501
            'button_ref': 'Reference',
            'videos': ['https://netflix-encoding-subjective-testing.s3.us-west-1.amazonaws.com/media/mp4/4kplus_study_demo/SolLevante_A/SolLevante_A__0_24_0_29__SdrVvhevceCrfVE__3820_2160__qp14_enable_audio_False_level_50_vmaf62.39_phonevmaf81.78_psnr22.02_kbps24650.51.mp4',  # noqa E501
                       'https://netflix-encoding-subjective-testing.s3.us-west-1.amazonaws.com/media/mp4/4kplus_study_demo/SMendesConcert_A/SMendesConcert_A__83_21.9_83_26.9__SdrVvhevceCrfVE__3820_2160__qp14_enable_audio_False_level_50_vmaf58.94_phonevmaf78.94_psnr25.44_kbps25447.21.mp4',  # noqa E501
                       ],
            'buttons': ['A', 'B'],
            'stimulusvotegroup_ids': [0, 1],
            'video_display_percentage': 100,
            'preload_videos': True,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    @method_decorator(never_cache)
    def demo_4kplus_with_preload_s3_b(self, request, extra_context=None):
        page = SamviqPage({
            'title': 'Round 1 of 10',
            'video_ref': 'https://netflix-encoding-subjective-testing.s3.us-west-1.amazonaws.com/media/mp4/4kplus_study_demo/LifeInColor_B/LifeInColor_B__3_22_3_27__SdrVvhevceCrfVE__3820_2160__qp14_enable_audio_False_level_51_vmaf63.88_phonevmaf83.20_psnr23.51_kbps39289.47.mp4',  # noqa E501
            'button_ref': 'Reference',
            'videos': ['https://netflix-encoding-subjective-testing.s3.us-west-1.amazonaws.com/media/mp4/4kplus_study_demo/SolLevante_A/SolLevante_A__0_24_0_29__SdrVvhevceCrfVE__3820_2160__qp14_enable_audio_False_level_50_vmaf62.39_phonevmaf81.78_psnr22.02_kbps24650.51.mp4',  # noqa E501
                       'https://netflix-encoding-subjective-testing.s3.us-west-1.amazonaws.com/media/mp4/4kplus_study_demo/SMendesConcert_A/SMendesConcert_A__83_21.9_83_26.9__SdrVvhevceCrfVE__3820_2160__qp14_enable_audio_False_level_50_vmaf58.94_phonevmaf78.94_psnr25.44_kbps25447.21.mp4',  # noqa E501
                       ],
            'buttons': ['A', 'B'],
            'stimulusvotegroup_ids': [0, 1],
            'video_display_percentage': 100,
            'preload_videos': True,
        })
        context = {
            **self.each_context(request),
            **page.context,
        }
        request.current_app = self.name
        return TemplateResponse(request, page.get_template(), context)

    def reset_session(self, request, session_id, extra_context=None):
        from .models import Experiment, Session
        session_id = int(session_id)
        session: Session = Session.objects.get(id=session_id)
        exp: Experiment = session.experiment
        ec = self._get_experiment_controller(exp, request)
        ec.reset_session(session)
        return HttpResponseRedirect(reverse('nest:status'))

    def start_session(self, request, session_id, extra_context=None):
        self._verify_subject(request, session_id)
        self._set_test_cookie(request)
        response = HttpResponseRedirect(
            reverse("nest:step_session", kwargs={'session_id': session_id}))
        self._clear_session_cookie(request)
        return response

    @staticmethod
    def _verify_subject(request, session_id):
        from .models import Session
        sess: Session = Session.objects.get(id=session_id)
        assert request.user.get_username() == sess.subject.user.username, \
            "expect subject with username {} for session {} but got username {}".format(
                sess.subject.user.username, session_id,
                request.user.get_username())

    def step_session(self, request, session_id, extra_context=None):  # noqa C901

        if not self._test_cookie_worked(request):
            title = 'Cookie disabled'
            proceed_url = reverse('nest:status')
            text_html = """ <p> To start the test, you must first enable cookie in your browser. </p> """  # noqa E501
            actions_html = f""" <p> <a class="button" href="{proceed_url}"  id="start">Back to Main Page</a> </p> """  # noqa E501
            page = GenericPage({'title': title, 'text_html': text_html, 'actions_html': actions_html})
            context = {**self.each_context(request), **page.context}
            request.current_app = self.name
            return TemplateResponse(request, page.get_template(), context)

        self._verify_subject(request, session_id)

        from .control import ExperimentController, SessionStatus
        from .models import Experiment, Session
        from .models import Round, StimulusGroup, StimulusVoteGroup, Vote
        session_id = int(session_id)
        session: Session = Session.objects.get(id=session_id)
        exp: Experiment = session.experiment
        ec: ExperimentController = self._get_experiment_controller(exp, request)

        VoteClass = Vote.find_subclass(ec.experiment_config.vote_scale)

        # only check session status (expensive) when GET
        if request.method == 'GET':

            start_time = time()
            ss: SessionStatus = ec.get_session_status(session)
            logger.info(f'get_session_status for session {session.id} took {time() - start_time} sec')
            if ss == SessionStatus.FINISHED:

                title = 'Session has already been completed'
                proceed_url = reverse('nest:status')
                text_html = """ <p> You have already completely this session ({}, ID {}). </p> """. \
                    format(session.experiment.title, session_id)
                actions_html = f""" <p> <a class="button" href="{proceed_url}"  id="start">Main page</a> </p>"""
                page = GenericPage({'title': title, 'text_html': text_html, 'actions_html': actions_html})
                context = {**self.each_context(request), **page.context}
                request.current_app = self.name
                return TemplateResponse(request, page.get_template(), context)

            else:
                assert ss in [SessionStatus.INITIALIZED,
                              SessionStatus.PARTIALLY_FINISHED]

            assert ss == SessionStatus.INITIALIZED, \
                "expect session {} status {}, but got: {}".format(
                    session, SessionStatus.INITIALIZED, ss)

        # figure out what's the next step, by comparing the steps_planned
        # obtained from the ec, and the steps_performed that is stored in the
        # cookie
        session_id_from_cookie = self._get_session_id_from_cookie(request)
        if session_id_from_cookie is not None:
            assert session_id_from_cookie == session_id
        steps_performed = self._get_session_steps_from_cookie(request)
        steps_planned = ec.get_session_steps(session)

        # do some basic sanity check
        assert len(steps_performed) <= len(steps_planned)
        for sperf, splan in zip(steps_performed, steps_planned):
            assert 'position' in sperf and 'position' in splan
            assert 'round_id' in sperf['position'] and 'round_id' in splan['position']
            assert sperf['position']['round_id'] == splan['position']['round_id']
            if 'before_or_after' in sperf['position']:
                assert 'before_or_after' in splan['position']
                assert sperf['position']['before_or_after'] == \
                       splan['position']['before_or_after']
            assert 'context' in splan
            if 'stimulusgroup_id' in splan['context']:
                assert 'context' in sperf and 'stimulusgroup_id' in sperf['context']
                assert sperf['context']['stimulusgroup_id'] == \
                       splan['context']['stimulusgroup_id']
            if 'before_or_after' in splan['position']:
                assert 'title' in splan['context']
                assert 'text_html' in splan['context']
                assert 'actions_html' in splan['context']
            # title, text_html, actions_html are ignored when step is addition

        if len(steps_performed) == len(steps_planned):
            # have done all steps. next:
            # 1) record results in db (by creating votes of round and svg, update round with response_sec)
            # 2) delete cookie
            # 3) display done page

            for step in steps_performed:
                if self._step_is_addition(step):
                    continue
                assert 'context' in step
                assert 'score' in step['context']
                assert 'response_sec' in step['context']
                rnd: Round = Round.objects.get(
                    session=session, round_id=step['position']['round_id'])

                if (ec.experiment_config.methodology == 'acr5c' and ec.experiment_config.vote_scale == '0_TO_100') or \
                        (ec.experiment_config.methodology in ['acr', 'dcr'] and
                         ec.experiment_config.vote_scale in [
                             'THREE_POINT',
                             'FIVE_POINT',
                             'SEVEN_POINT',
                             'ELEVEN_POINT'
                         ]) or \
                        (ec.experiment_config.methodology == 'tafc' and
                         ec.experiment_config.vote_scale == '2AFC') or \
                        (ec.experiment_config.methodology == 'ccr' and
                         ec.experiment_config.vote_scale == 'CCR_THREE_POINT') or \
                        (ec.experiment_config.methodology == 'samviq' and
                         ec.experiment_config.vote_scale == '0_TO_100') or \
                        (ec.experiment_config.methodology == 'samviq5d' and
                         ec.experiment_config.vote_scale == 'FIVE_POINT'):
                    sg: StimulusGroup = StimulusGroup.objects.get(
                        experiment=exp,
                        stimulusgroup_id=step['context']['stimulusgroup_id'])
                    score_dict = step['context']['score']
                    assert isinstance(score_dict, dict)
                    for svgid, score in score_dict.items():
                        assert isinstance(score, int)
                        svg: StimulusVoteGroup = StimulusVoteGroup.objects.get(
                            stimulusgroup=sg, stimulusvotegroup_id=svgid)

                        # in each round, for a single svg, there can only be one
                        # vote; if more than one, it could be a duplicated submission
                        try:
                            vote2: VoteClass = VoteClass.objects.get(
                                round=rnd, stimulusvotegroup=svg)
                            if vote2.score == score:
                                msg = f'skip saving vote {score} as vote with ' \
                                      f'score {vote2.score} and svg {svg} already ' \
                                      f'exists in round {rnd}, possibly a double ' \
                                      f'POST submission.'
                                logger.warning(msg)
                                continue
                            else:
                                msg = f'error saving vote {score} as vote with ' \
                                      f'score {vote2.score} and svg {svg} already ' \
                                      f'exists in round {rnd}.'
                                logger.error(msg)
                                raise AssertionError(msg)
                        except VoteClass.DoesNotExist:
                            pass

                        vote: VoteClass = VoteClass(score=score,
                                                    round=rnd,
                                                    stimulusvotegroup=svg)
                        vote.save()
                else:
                    assert False, 'The combination of {m} methodology with {s} vote_scale is undefined'.format(
                        m=ec.experiment_config.methodology, s=ec.experiment_config.vote_scale)

                response_sec = step['context']['response_sec']
                assert response_sec != 'none'
                rnd.response_sec = response_sec
                rnd.save()

            title = 'Test done'
            proceed_url = reverse('nest:status')
            if 'text_html' in ec.experiment_config.done_context:
                text_html = ec.experiment_config.done_context['text_html']
            else:
                text_html = """ <p> You have completed the test. </p> """
            actions_html = f""" <p> <a class="button" href="{proceed_url}"  id="start">Done</a> </p>"""
            page = GenericPage({'title': title, 'text_html': text_html, 'actions_html': actions_html})
            context = {**self.each_context(request), **page.context}
            request.current_app = self.name
            response = TemplateResponse(request, page.get_template(), context)

            self._clear_test_cookie(request)
            self._clear_session_cookie(request)

            return response

        next_step = steps_planned[len(steps_performed)]
        step_is_addition = self._step_is_addition(next_step)
        if step_is_addition:
            action_html_template = next_step['context']['actions_html']
            assert '{action_url}' in action_html_template
            action_html = action_html_template.format(
                action_url=reverse("nest:step_session", kwargs={'session_id': session_id}))
            page = GenericPage({'title': next_step['context']['title'],
                                'text_html': next_step['context']['text_html'],
                                'actions_html': action_html})
            context = {**self.each_context(request), **page.context}
            request.current_app = self.name
            response = TemplateResponse(request, page.get_template(), context)

            del next_step['context']  # avoid context (title...) in cookie
            self._set_session_cookie(request, {
                'steps': steps_performed + [next_step],
                'session_id': session_id})

        else:  # step is actual round

            if request.method == 'GET':

                training_round_ids = ec.experiment_config.training_round_ids
                if next_step['position']['round_id'] in training_round_ids:
                    title = "Training {} of {}".format(
                        next_step['position']['round_id'] + 1,
                        len(training_round_ids))
                else:
                    title = "Round {} of {}".format(
                        next_step['position']['round_id'] + 1 - len(training_round_ids),
                        ec.experiment_config.rounds_per_session - len(training_round_ids))

                round_start_sec: float = time()
                rnd: Round = Round.objects.get(
                    session=session, round_id=next_step['position']['round_id'])
                round_start_sec_in_cookie = self._get_round_response_sec_from_cookie(request, rnd.id)
                if round_start_sec_in_cookie is not None:
                    logger.warning(f'round_start_sec {round_start_sec_in_cookie} for round {rnd.id} '
                                   f'in cookie exists, override with {round_start_sec}')
                self._set_round_response_sec_in_cookie(request, rnd.id, round_start_sec)

                if (ec.experiment_config.methodology in ['acr', 'dcr']
                        and ec.experiment_config.vote_scale in [
                            'THREE_POINT',
                            'FIVE_POINT',
                            'SEVEN_POINT',
                            'ELEVEN_POINT',
                        ]) or (ec.experiment_config.methodology == 'acr5c' and
                               ec.experiment_config.vote_scale == '0_TO_100'):

                    PageClass = map_methodology_to_page_class(
                        ec.experiment_config.methodology)
                    sgid: int = next_step['context']['stimulusgroup_id']
                    video_display_percentage = ec.experiment_config. \
                        stimulus_config.get_video_display_percentage(sgid)
                    assert video_display_percentage is not None

                    svgid = self._get_matched_single_stimulusvotegroup_id(ec, next_step)

                    d = {'title': title,
                         'session_id': session_id,
                         'video_display_percentage': video_display_percentage,
                         'stimulusvotegroup_id': svgid,
                         **ec.experiment_config.round_context,
                         }

                    if ec.experiment_config.methodology.startswith('acr'):
                        sid = self._get_matched_single_stimulus_id(ec, svgid)
                        s = self._get_matched_stimulus_dict(ec, sid)
                        assert s['type'] == 'video/mp4'
                        d['video'] = s['path']

                    elif ec.experiment_config.methodology == 'dcr':
                        dis_sid, ref_sid = self._get_matched_double_stimulus_ids(ec, svgid)
                        dis_s = self._get_matched_stimulus_dict(ec, dis_sid)
                        ref_s = self._get_matched_stimulus_dict(ec, ref_sid)
                        assert dis_s['type'] == 'video/mp4'
                        assert ref_s['type'] == 'video/mp4'
                        d['video_a'] = ref_s['path']
                        d['video_b'] = dis_s['path']

                    else:
                        assert False

                    if ec.experiment_config.vote_scale == 'THREE_POINT':
                        # only if choices not yet overridden by round_context:
                        if 'choices' not in d:
                            d['choices'] = \
                                ["Indistinguishable",
                                 "Distinguishable but acceptable as premium quality",
                                 "Not acceptable as premium quality",
                                 ]
                    elif ec.experiment_config.vote_scale in ['FIVE_POINT', '0_TO_100']:
                        pass
                    elif ec.experiment_config.vote_scale == 'SEVEN_POINT':
                        # only if choices not yet overridden by round_context:
                        if 'choices' not in d:
                            d['choices'] = \
                                ['7 - Imperceptible',
                                 '6 - Slightly perceptible',
                                 '5 - Perceptible',
                                 '4 - Clearly perceptible',
                                 '3 - Annoying',
                                 '2 - Severely annoying',
                                 '1 - Unwatchable']
                    elif ec.experiment_config.vote_scale == 'ELEVEN_POINT':
                        # only if choices not yet overridden by round_context:
                        if 'choices' not in d:
                            d['choices'] = \
                                ['11 - Imperceptible',
                                 '10 - Slightly perceptible somewhere',
                                 '9 - Slightly perceptible everywhere',
                                 '8 - Perceptible somewhere',
                                 '7 - Perceptible everywhere',
                                 '6 - Clearly perceptible somewhere',
                                 '5 - Clearly perceptible everywhere',
                                 '4 - Annoying somewhere',
                                 '3 - Annoying everywhere',
                                 '2 - Severely annoying somewhere',
                                 '1 - Severely annoying everywhere']

                    else:
                        assert False
                    page = PageClass(d)
                    context = {**self.each_context(request), **page.context}
                    request.current_app = self.name
                    response = TemplateResponse(request, page.get_template(), context)
                elif (ec.experiment_config.methodology == 'ccr' and
                      ec.experiment_config.vote_scale == 'CCR_THREE_POINT') \
                        or (ec.experiment_config.methodology == 'tafc' and
                            ec.experiment_config.vote_scale == '2AFC'):

                    VoteClass = Vote.find_subclass(ec.experiment_config.vote_scale)

                    svgid = self._get_matched_single_stimulusvotegroup_id(ec, next_step)
                    sid_1st, sid_2nd = self._get_matched_double_stimulus_ids(ec, svgid)
                    s_1st = self._get_matched_stimulus_dict(ec, sid_1st)
                    s_2nd = self._get_matched_stimulus_dict(ec, sid_2nd)
                    assert s_1st['type'] == 'video/mp4'
                    assert s_2nd['type'] == 'video/mp4'

                    # randomize the order of stimuli on 2AFC page
                    if random.random() < 0.5:
                        video_a = s_1st['path']
                        video_b = s_2nd['path']
                        video_a_to_b_values = VoteClass.support
                    else:
                        video_b = s_1st['path']
                        video_a = s_2nd['path']
                        video_a_to_b_values = list(reversed(VoteClass.support))

                    sgid: int = next_step['context']['stimulusgroup_id']
                    video_display_percentage = ec.experiment_config. \
                        stimulus_config.get_video_display_percentage(sgid)
                    assert video_display_percentage is not None
                    d = {'title': title,
                         'video_a': video_a,
                         'video_b': video_b,
                         'video_a_to_b_values': video_a_to_b_values,
                         'session_id': session_id,
                         'video_display_percentage': video_display_percentage,
                         'stimulusvotegroup_id': svgid,
                         **ec.experiment_config.round_context,
                         }
                    PageClass = map_methodology_to_page_class(
                        ec.experiment_config.methodology)
                    page = PageClass(d)
                    context = {**self.each_context(request), **page.context}
                    request.current_app = self.name
                    response = TemplateResponse(request, page.get_template(), context)
                elif (ec.experiment_config.methodology == 'samviq5d' and
                      ec.experiment_config.vote_scale == 'FIVE_POINT') or \
                        (ec.experiment_config.methodology == 'samviq' and
                         ec.experiment_config.vote_scale == '0_TO_100'):
                    svgids = self._get_matched_stimulusvotegroup_ids(ec, next_step)

                    # randomize the order of stimuli on the SAMVIQ page
                    random.shuffle(svgids)

                    ref_sid = None
                    dis_sids = []
                    for svgid in svgids:
                        sid, sid2 = self._get_matched_double_stimulus_ids(ec, svgid)
                        if ref_sid is None:
                            ref_sid = sid2
                        else:
                            assert ref_sid == sid2
                        dis_sids.append(sid)
                    ref_s = self._get_matched_stimulus_dict(ec, ref_sid)
                    dis_ss = [self._get_matched_stimulus_dict(ec, dis_sid)
                              for dis_sid in dis_sids]
                    assert ref_s['type'] == 'video/mp4'
                    for dis_s in dis_ss:
                        assert dis_s['type'] == 'video/mp4'
                    PageClass = map_methodology_to_page_class(
                        ec.experiment_config.methodology)
                    sgid: int = next_step['context']['stimulusgroup_id']
                    video_display_percentage = ec.experiment_config. \
                        stimulus_config.get_video_display_percentage(sgid)
                    assert video_display_percentage is not None
                    d = {
                        'title': title,
                        'video_ref': ref_s['path'],
                        'button_ref': 'Reference',
                        'videos': [dis_s['path'] for dis_s in dis_ss],
                        'stimulusvotegroup_ids': svgids,
                        'buttons': list(string.ascii_uppercase[:len(dis_ss)]),
                        'video_display_percentage': video_display_percentage,
                        **ec.experiment_config.round_context,
                    }
                    page = PageClass(d)
                    context = {
                        **self.each_context(request),
                        **page.context,
                    }
                    request.current_app = self.name
                    return TemplateResponse(request, page.get_template(), context)
                else:
                    assert False, 'The combination of {m} methodology with {s} vote_scale is undefined'.format(
                        m=ec.experiment_config.methodology, s=ec.experiment_config.vote_scale)

            elif request.method == 'POST':

                round_end_sec: float = time()
                rnd: Round = Round.objects.get(
                    session=session, round_id=next_step['position']['round_id'])
                round_start_sec_in_cookie = self._get_round_response_sec_from_cookie(request, rnd.id)
                try:
                    assert round_start_sec_in_cookie is not None, \
                        "expect round_start_sec in cookie but it's not"
                    response_sec = round_end_sec - round_start_sec_in_cookie
                    assert response_sec > 0, \
                        f'expect response_sec > 0 but got {response_sec} ' \
                        f'(round_end_sec {round_end_sec}, round_start_sec_in_cookie {round_start_sec_in_cookie})'
                except AssertionError as e:
                    if str(e) == "expect round_start_sec in cookie but it's not":
                        logger.warning(str(e))
                        response_sec = 'none'
                    else:
                        raise e

                sgid = next_step['context']['stimulusgroup_id']
                sg: StimulusGroup = StimulusGroup.objects.get(
                    experiment=exp,
                    stimulusgroup_id=sgid)
                skip_set_cookie: bool = False

                if (ec.experiment_config.methodology == 'acr' and
                    ec.experiment_config.vote_scale == 'FIVE_POINT') or \
                        (ec.experiment_config.methodology == 'acr5c' and
                         ec.experiment_config.vote_scale == '0_TO_100') or \
                        (ec.experiment_config.methodology == 'dcr' and
                         ec.experiment_config.vote_scale in [
                             'THREE_POINT',
                             'FIVE_POINT',
                             'SEVEN_POINT',
                             'ELEVEN_POINT',
                         ]) or \
                        (ec.experiment_config.methodology == 'tafc' and
                         ec.experiment_config.vote_scale == '2AFC') or \
                        (ec.experiment_config.methodology == 'ccr' and
                         ec.experiment_config.vote_scale == 'CCR_THREE_POINT') or \
                        (ec.experiment_config.methodology == 'samviq5d' and
                         ec.experiment_config.vote_scale == 'FIVE_POINT') or \
                        (ec.experiment_config.methodology == 'samviq' and
                         ec.experiment_config.vote_scale == '0_TO_100'):
                    score_dict = dict()
                    for key, val in request.POST.items():
                        mo = re.match(
                            r"^{m}_([0-9]*)$".format(m=map_methodology_to_html_id_key(ec.experiment_config.methodology)),
                            key)
                        if mo is None:
                            continue
                        svgid = int(mo.group(1))
                        try:
                            StimulusVoteGroup.objects.get(
                                stimulusgroup=sg, stimulusvotegroup_id=svgid)
                        except StimulusVoteGroup.DoesNotExist:
                            # verify svg does exist; if not exist, it could
                            # be double POST submission
                            skip_set_cookie = True
                            logger.warning(f'skip setting cookie as stimulus_vote_group '
                                           f'{svgid} with stimulus_group {sgid} could not '
                                           f'be identified, possibly a double POST submission.')
                            break
                        score = int(val)
                        VoteClass.assert_vote(score)
                        score_dict[svgid] = score
                    if skip_set_cookie is True:
                        pass
                    else:
                        next_step['context']['score'] = score_dict
                        next_step['context']['response_sec'] = response_sec
                        self._set_session_cookie(request, {
                            'steps': steps_performed + [next_step],
                            'session_id': session_id})
                    response = HttpResponseRedirect(
                        reverse("nest:step_session", kwargs={'session_id': session_id}))
                else:
                    assert False, 'The combination of {m} methodology with {s} vote_scale is undefined'.format(
                        m=ec.experiment_config.methodology, s=ec.experiment_config.vote_scale)
            else:
                assert False

        return response

    @staticmethod
    def _get_matched_stimulus_dict(ec, sid):
        s: dict
        for s in ec.experiment_config.stimulus_config.stimuli:
            if s['stimulus_id'] == sid:
                break
        else:
            assert False, 'no stimilus with matching ' \
                          'stimulus_id {} found: {}'. \
                format(sid,
                       ec.experiment_config.stimulus_config.stimuli)
        return s

    @staticmethod
    def _get_matched_single_stimulus_id(ec, svgid):
        svg: dict
        for svg in ec.experiment_config.stimulus_config.stimulusvotegroups:
            if svg['stimulusvotegroup_id'] == svgid:
                break
        else:
            assert False, 'no stimulusvotegroup with matching ' \
                          'stimulusvotegroup_id {} found: {}'. \
                format(svgid,
                       ec.experiment_config.stimulus_config.stimulusvotegroups)
        assert len(svg['stimulus_ids']) == 1, \
            "expect only one stimulus per" \
            " stimulusvotegroup, but has {}".format(svg['stimulus_ids'])
        sid = svg['stimulus_ids'][0]
        return sid

    @staticmethod
    def _get_matched_double_stimulus_ids(ec, svgid):
        svg: dict
        for svg in ec.experiment_config.stimulus_config.stimulusvotegroups:
            if svg['stimulusvotegroup_id'] == svgid:
                break
        else:
            assert False, 'no stimulusvotegroup with matching ' \
                          'stimulusvotegroup_id {} found: {}'. \
                format(svgid,
                       ec.experiment_config.stimulus_config.stimulusvotegroups)
        assert len(svg['stimulus_ids']) == 2, \
            "expect exactly two stimuli per" \
            " stimulusvotegroup, but has {}".format(svg['stimulus_ids'])
        sid = svg['stimulus_ids'][0]
        sid2 = svg['stimulus_ids'][1]
        return sid, sid2

    @staticmethod
    def _get_matched_single_stimulusvotegroup_id(ec, step):
        sg: dict
        for sg in ec.experiment_config.stimulus_config.stimulusgroups:
            if sg['stimulusgroup_id'] == step['context']['stimulusgroup_id']:
                break
        else:
            assert False, 'no stimulusgroup with matching ' \
                          'stimulusgroup_id {} found: {}'. \
                format(step['context']['stimulusgroup_id'],
                       ec.experiment_config.stimulus_config.stimulusgroups)
        assert len(sg['stimulusvotegroup_ids']) == 1, \
            "expect only one stimulusvotegroup per" \
            " round, but has {}".format(sg['stimulusvotegroup_ids'])
        svgid = sg['stimulusvotegroup_ids'][0]
        return svgid

    @classmethod
    def _get_matched_stimulusvotegroup_ids(cls, ec, step):
        return cls._get_matched_stimulusgroup(ec, step)['stimulusvotegroup_ids']

    @staticmethod
    def _get_matched_stimulusgroup(ec, step):
        sg: dict
        for sg in ec.experiment_config.stimulus_config.stimulusgroups:
            if sg['stimulusgroup_id'] == step['context']['stimulusgroup_id']:
                break
        else:
            assert False, 'no stimulusgroup with matching ' \
                          'stimulusgroup_id {} found: {}'. \
                format(step['context']['stimulusgroup_id'],
                       ec.experiment_config.stimulus_config.stimulusgroups)
        return sg

    @staticmethod
    def _step_is_addition(step):
        """
        Criterion to determine if a step is addition (but not round).

        A step could be either an "addition step" (instruction, survey, etc.)
        or "round step" (step where some stimuli got voted).
        """
        return 'before_or_after' in step['position']


class DefaultNestSite(LazyObject):
    def _setup(self):
        NestSiteClass = import_string(apps.get_app_config('nest').default_site)
        self._wrapped = NestSiteClass()


site = DefaultNestSite()

nest_site/nest/sites.py
Netflix/e2nest
import logging

from django.contrib.auth.models import User
from django.test import TestCase
from django.urls import reverse
from nest.config import NestConfig
from nest.io import ExperimentUtils
from nest.models import Subject


class TestViews(TestCase):

    def setUp(self) -> None:
        self.user = User.objects.create_user('user', password='pass', is_staff=True)
        logging.disable(logging.CRITICAL)

    def tearDown(self):
        self.user.delete()
        logging.disable(logging.NOTSET)

    def test_login(self):
        response = self.client.get(reverse('admin:login'))
        self.assertEqual(response.status_code, 200)

    def test_logout(self):
        response = self.client.get(reverse('admin:logout'))
        self.assertEqual(response.status_code, 302)

    def test_nestexp_with_login(self):
        response = self.client.get(reverse('admin:nestexp'))
        self.assertEqual(response.status_code, 302)

        login = self.client.login(username='user', password='paxx')
        self.assertFalse(login)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('admin:nestexp'))
        self.assertEqual(response.status_code, 200)

    def test_nestexp(self):
        self.client.login(username='user', password='pass')
        response = self.client.get(reverse('admin:nestexp'))
        self.assertEqual(response.status_code, 200)

    def test_nest_download_sureal(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        self.client.login(username='user', password='pass')

        self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'acr': '1'})
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'acr': '4'})
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.client.login(username='staff', password='pass')
        response = self.client.get(reverse('admin:download_sureal', args=(1,)))
        self.assertEqual(response.status_code, 200)

nest_site/nest/tests/admin_view_tests.py
Netflix/e2nest
import json

from django.test import TestCase
from nest.config import ExperimentConfig, NestConfig, StimulusConfig


class TestStimulusConfig(TestCase):

    def test_stim_config(self):
        scfg = StimulusConfig({
            "contents": [
                {
                    "content_id": 0,
                    "name": "CTS3E1_B__15_55_16_0",
                }
            ],
            "stimuli": [
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4",  # noqa E501
                    "stimulus_id": 0,
                    "type": "video/mp4",
                    "content_id": 0,
                },
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4",  # noqa E501
                    "stimulus_id": 1,
                    "type": "video/mp4",
                    "content_id": 0,
                }
            ],
            "stimulusvotegroups": [
                {
                    "stimulus_ids": [
                        0
                    ],
                    "stimulusvotegroup_id": 0
                },
                {
                    "stimulus_ids": [
                        1
                    ],
                    "stimulusvotegroup_id": 1
                }
            ],
            'stimulusgroups': [
                {'info': {'flavors': ['training']}, 'stimulusgroup_id': 0, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 4, 'stimulusvotegroup_ids': [1]},
                {'stimulusgroup_id': 2, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 3, 'stimulusvotegroup_ids': [0], 'video_display_percentage': 30},
                {'info': {'flavors': ['decoy']}, 'stimulusgroup_id': 5, 'stimulusvotegroup_ids': [0]},
            ],
        })
        self.assertEqual(len(scfg.stimulusgroups), 5)
        self.assertEqual(scfg.get_video_display_percentage(3), 30)
        self.assertEqual(scfg.get_video_display_percentage(4), 100)
        self.assertEqual(scfg.get_video_display_percentage(5), 100)
        self.assertEqual(scfg.get_video_display_percentage(6), None)

    def test_stim_config_with_public_path(self):
        scfg = StimulusConfig({
            "contents": [
                {
                    "content_id": 0,
                    "name": "CTS3E1_B__15_55_16_0",
                }
            ],
            "stimuli": [
                {
                    "path": "http://media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4",  # noqa E501
                    "stimulus_id": 0,
                    "type": "video/mp4",
                    "content_id": 0,
                },
                {
                    "path": "https://media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4",  # noqa E501
                    "stimulus_id": 1,
                    "type": "video/mp4",
                    "content_id": 0,
                }
            ],
            "stimulusvotegroups": [
                {
                    "stimulus_ids": [
                        0
                    ],
                    "stimulusvotegroup_id": 0
                },
                {
                    "stimulus_ids": [
                        1
                    ],
                    "stimulusvotegroup_id": 1
                }
            ],
            'stimulusgroups': [
                {'info': {'flavors': ['training']}, 'stimulusgroup_id': 0, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 4, 'stimulusvotegroup_ids': [1]},
                {'stimulusgroup_id': 2, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 3, 'stimulusvotegroup_ids': [0], 'video_display_percentage': 30},
                {'info': {'flavors': ['decoy']}, 'stimulusgroup_id': 5, 'stimulusvotegroup_ids': [0]},
            ],
        })
        self.assertEqual(len(scfg.stimulusgroups), 5)
        self.assertEqual(scfg.get_video_display_percentage(3), 30)
        self.assertEqual(scfg.get_video_display_percentage(4), 100)
        self.assertEqual(scfg.get_video_display_percentage(5), 100)
        self.assertEqual(scfg.get_video_display_percentage(6), None)


class TestExperimentConfig(TestCase):

    def setUp(self) -> None:
        self.scfg = StimulusConfig({
            "contents": [
                {
                    "content_id": 0,
                    "name": "CTS3E1_B__15_55_16_0",
                }
            ],
            "stimuli": [
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4",  # noqa E501
                    "stimulus_id": 0,
                    "type": "video/mp4",
                    "content_id": 0,
                },
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4",  # noqa E501
                    "stimulus_id": 1,
                    "type": "video/mp4",
                    "content_id": 0,
                }
            ],
            "stimulusvotegroups": [
                {
                    "stimulus_ids": [
                        0
                    ],
                    "stimulusvotegroup_id": 0
                },
                {
                    "stimulus_ids": [
                        1
                    ],
                    "stimulusvotegroup_id": 1
                }
            ],
            'stimulusgroups': [
                {'info': {'flavors': ['training']}, 'stimulusgroup_id': 0, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 4, 'stimulusvotegroup_ids': [1]},
                {'stimulusgroup_id': 2, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 3, 'stimulusvotegroup_ids': [0]},
                {'info': {'flavors': ['decoy']}, 'stimulusgroup_id': 5, 'stimulusvotegroup_ids': [0]},
            ],
        })

    def test_exp_config_0(self):
        ecfg = ExperimentConfig(
            stimulus_config=self.scfg,
            config={
                'title': 'test',
                'description': 'test',
                'methodology': 'acr',
                'vote_scale': 'FIVE_POINT',
                'rounds_per_session': 10,
            }
        )
        self.assertEqual(ecfg.rounds_per_session, 10)
        self.assertEqual(ecfg.random_seed, None)
        self.assertEqual(len(ecfg.prioritized), 0)
        self.assertEqual(ecfg.vote_scale, 'FIVE_POINT')

    def test_exp_config_known_vote_scale(self):
        with self.assertRaises(AssertionError):
            ExperimentConfig(stimulus_config=self.scfg,
                             config={'vote_scale': 'FIVE_POINT_XXX'})

    def test_exp_config(self):
        ecfg = ExperimentConfig(
            stimulus_config=self.scfg,
            config={
                'title': 'test',
                'description': 'test',
                'rounds_per_session': 8,
                'random_seed': 3,
                'prioritized': [
                    {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},
                    {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
                    {'session_idx': 1, 'round_id': 5, 'stimulusgroup_id': 4},
                    {'session_idx': 2, 'round_id': 5, 'stimulusgroup_id': 4},
                    {'session_idx': 2, 'round_id': None, 'stimulusgroup_id': 2},
                ],
                'blocklist_stimulusgroup_ids': [0],
                'training_round_ids': [0],
                'vote_scale': '0_TO_100',
                'methodology': 'acr5c',
            })
        self.assertEqual(ecfg.rounds_per_session, 8)
        self.assertEqual(ecfg.random_seed, 3)
        self.assertEqual(len(ecfg.prioritized), 5)
        self.assertEqual(len(ecfg.blocklist_stimulusgroup_ids), 1)
        self.assertEqual(len(ecfg.training_round_ids), 1)
        self.assertEqual(ecfg.vote_scale, '0_TO_100')

    def test_exp_order_sgid_outofbound(self):
        scfg = StimulusConfig({
            "contents": [
                {
                    "content_id": 0,
                    "name": "CTS3E1_B__15_55_16_0",
                }
            ],
            "stimuli": [
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4",  # noqa E501
                    "stimulus_id": 0,
                    "type": "video/mp4",
                    "content_id": 0,
                },
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4",  # noqa E501
                    "stimulus_id": 1,
                    "type": "video/mp4",
                    "content_id": 0,
                }
            ],
            "stimulusvotegroups": [
                {
                    "stimulus_ids": [
                        0
                    ],
                    "stimulusvotegroup_id": 0
                },
                {
                    "stimulus_ids": [
                        1
                    ],
                    "stimulusvotegroup_id": 1
                }
            ],
            'stimulusgroups': [
                {'info': {'flavors': ['training']}, 'stimulusgroup_id': 0, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 1, 'stimulusvotegroup_ids': [1]},
                {'stimulusgroup_id': 2, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 3, 'stimulusvotegroup_ids': [0]},
                {'info': {'flavors': ['decoy']}, 'stimulusgroup_id': 4, 'stimulusvotegroup_ids': [0]},
            ],
        })
        with self.assertRaises(AssertionError):
            ExperimentConfig(
                stimulus_config=scfg,
                config={
                    'title': 'test',
                    'description': 'test',
                    'rounds_per_session': 8,
                    'random_seed': 3,
                    'prioritized': [
                        {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},
                        {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 5},
                        {'session_idx': 1, 'round_id': 5, 'stimulusgroup_id': 5},
                        {'session_idx': 2, 'round_id': 5, 'stimulusgroup_id': 5},
                    ]
                })

    def test_exp_additions(self):
        config_filepath = NestConfig.tests_resource_path('cvxhull_subjexp_toy_x.json')
        with open(config_filepath, 'rt') as fp:
            config = json.load(fp)
        scfg = StimulusConfig(config['stimulus_config'])
        ExperimentConfig(stimulus_config=scfg,
                         config=config['experiment_config'])

    def test_exp_additions_dcr(self):
        config_filepath = NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr.json')
        with open(config_filepath, 'rt') as fp:
            config = json.load(fp)
        scfg = StimulusConfig(config['stimulus_config'])
        ExperimentConfig(stimulus_config=scfg,
                         config=config['experiment_config'])

    def test_exp_additions_mlds(self):
        config_filepath = NestConfig.tests_resource_path('cvxhull_subjexp_toy_mlds.json')
        with open(config_filepath, 'rt') as fp:
            config = json.load(fp)
        # currently does not support svg involving 4 stimuli (e.g. MLDS)
        with self.assertRaises(AssertionError):
            StimulusConfig(config['stimulus_config'])

    def test_validate_html(self):
        config_filepath = NestConfig.tests_resource_path('cvxhull_subjexp_toy_w_invalid_done_context.json')
        with open(config_filepath, 'rt') as fp:
            config = json.load(fp)
        scfg = StimulusConfig(config['stimulus_config'])
        with self.assertRaises(AssertionError):
            ExperimentConfig(stimulus_config=scfg,
                             config=config['experiment_config'])

nest_site/nest/tests/config_tests.py
Netflix/e2nest
import json
import random

import numpy as np
from django.test import TestCase
from nest.config import ExperimentConfig, NestConfig, StimulusConfig
from nest.control import ExperimentController, SessionStatus
from nest.helpers import indices
from nest.models import Content, Experiment, Round, Session, Stimulus, \
    StimulusGroup, StimulusVoteGroup, Subject


class TestOrder(TestCase):

    def test_random_seed(self):
        d = ExperimentController._order(
            rounds_per_session=8,
            stimulusgroup_ids=[4, 3, 2, 1, 0],
            subject_id=1,
            ordering_so_far=list(),
            prioritized=list(),
            random_seed=3,
            blocklist_stimulusgroup_ids=list(),
        )
        self.assertEqual(d, {0: 4,
                             1: 0,
                             2: 1,
                             3: 2,
                             4: 1,
                             5: 0,
                             6: 4,
                             7: 3})

    def test_partial_prioritized(self):
        prioritized = [
            {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 1, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 2, 'round_id': 5, 'stimulusgroup_id': 4},
        ]

        d = ExperimentController._order(
            rounds_per_session=8,
            stimulusgroup_ids=list(range(5)),
            subject_id=1,
            ordering_so_far=list(),
            prioritized=prioritized,
            random_seed=3,
            blocklist_stimulusgroup_ids=list(),
        )
        self.assertEqual(d, {0: 0,
                             1: 2,
                             2: 3,
                             3: 3,
                             4: 1,
                             5: 4,
                             6: 0,
                             7: 4})

    def test_partial_prioritized2(self):
        prioritized = [
            {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': None, 'stimulusgroup_id': 0},
            {'session_idx': 1, 'round_id': 5, 'stimulusgroup_id': 0},
            {'session_idx': 2, 'round_id': 5, 'stimulusgroup_id': 0},
        ]

        d = ExperimentController._order(
            rounds_per_session=8,
            stimulusgroup_ids=[4, 3, 2, 1, 0],
            subject_id=1,
            ordering_so_far=list(),
            prioritized=prioritized,
            random_seed=3,
            blocklist_stimulusgroup_ids=list(),
        )
        # 'round_id': None means it could randomly appear among the rounds
        self.assertEqual(d, {0: 4,
                             1: 3,
                             2: 1,
                             3: 2,
                             4: 0,
                             5: 0,
                             6: 4,
                             7: 1})

    def test_partial_prioritized3(self):
        prioritized = [
            {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': None, 'stimulusgroup_id': 0},
            {'session_idx': 1, 'round_id': 5, 'stimulusgroup_id': 0},
            {'session_idx': 2, 'round_id': 5, 'stimulusgroup_id': 0},
        ]
        blacklist_sgids = [0]

        d = ExperimentController._order(
            rounds_per_session=8,
            stimulusgroup_ids=[4, 3, 2, 1, 0],
            subject_id=1,
            ordering_so_far=list(),
            prioritized=prioritized,
            random_seed=3,
            blocklist_stimulusgroup_ids=blacklist_sgids,
        )
        # 'round_id': None means it could randomly appear among the rounds
        # included in blocklist_stimulusgroup_ids means it won't appears in
        # other rounds rather than specified in the prioritized list
        self.assertEqual(d, {0: 4,
                             1: 1,
                             2: 2,
                             3: 3,
                             4: 1,
                             5: 0,
                             6: 4,
                             7: 2})

    def test_full_prioritized(self):
        prioritized = [
            {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},
            {'session_idx': 0, 'round_id': 1, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': 2, 'stimulusgroup_id': 2},
            {'session_idx': 0, 'round_id': 3, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': 4, 'stimulusgroup_id': 2},
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': 6, 'stimulusgroup_id': 2},
            {'session_idx': 0, 'round_id': 7, 'stimulusgroup_id': 4},
        ]

        d = ExperimentController._order(
            rounds_per_session=8,
            stimulusgroup_ids=list(range(5)),
            subject_id=1,
            ordering_so_far=list(),
            prioritized=prioritized,
            random_seed=3,
            blocklist_stimulusgroup_ids=list(),
        )
        self.assertEqual(d, {0: 0,
                             1: 4,
                             2: 2,
                             3: 4,
                             4: 2,
                             5: 4,
                             6: 2,
                             7: 4})

    def test_ordering_so_far(self):
        ordering_so_far = [
            {'subject': 0, 'stimulusgroups': {0: 1,
                                              1: 4,
                                              2: 4,
                                              3: 1,
                                              4: 2,
                                              5: 4,
                                              6: 3,
                                              7: 4}},
            {'subject': 1, 'stimulusgroups': {0: 0,
                                              1: 4,
                                              2: 2,
                                              3: 4,
                                              4: 2,
                                              5: 4,
                                              6: 2,
                                              7: 4}},
        ]
        d = ExperimentController._order(
            rounds_per_session=8,
            stimulusgroup_ids=list(range(5)),
            subject_id=2,
            ordering_so_far=ordering_so_far,
            prioritized=list(),
            random_seed=3,
            blocklist_stimulusgroup_ids=list(),
        )
        self.assertEqual(d, {0: 3,
                             1: 1,
                             2: 1,
                             3: 0,
                             4: 3,
                             5: 2,
                             6: 4,
                             7: 0})

    def test_partial_prioritized_repeated_element(self):
        prioritized = [
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 3},
        ]

        d = ExperimentController._order(
            rounds_per_session=8,
            stimulusgroup_ids=list(range(5)),
            subject_id=1,
            ordering_so_far=list(),
            prioritized=prioritized,
            random_seed=3,
            blocklist_stimulusgroup_ids=list(),
        )
        self.assertEqual(d, {0: 4,
                             1: 4,
                             2: 0,
                             3: 1,
                             4: 3,
                             5: 3,
                             6: 2,
                             7: 2})

    def test_partial_prioritized_training(self):
        # to set sg 0 as training:
        # 1) add it to the prioritized list
        # 2) mark it in the blocklist_stimulusgroup_ids,
        #    s.t. it won't appear elsewhere
        prioritized = [
            {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},
            {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 4},
        ]

        d = ExperimentController._order(
            rounds_per_session=8,
            stimulusgroup_ids=list(range(5)),
            subject_id=1,
            ordering_so_far=list(),
            prioritized=prioritized,
            random_seed=3,
            blocklist_stimulusgroup_ids=[0],
        )
        self.assertEqual(d, {0: 0,
                             1: 2,
                             2: 4,
                             3: 3,
                             4: 1,
                             5: 4,
                             6: 2,
                             7: 3})

    def test_split_list_two_different_subjects(self):
        pl = list()
        d1 = ExperimentController._order(
            rounds_per_session=5,
            stimulusgroup_ids=list(range(10)),
            subject_id=1,
            ordering_so_far=pl,
            prioritized=list(),
            random_seed=3,
            blocklist_stimulusgroup_ids=list(),
        )
        self.assertEqual(d1, {0: 9,
                              1: 7,
                              2: 2,
                              3: 3,
                              4: 4})
        pl.append({'subject': 1, 'stimulusgroups': d1})
        d2 = ExperimentController._order(
            rounds_per_session=5,
            stimulusgroup_ids=list(range(10)),
            subject_id=2,
            ordering_so_far=pl,
            prioritized=list(),
            random_seed=3,
            blocklist_stimulusgroup_ids=list(),
        )
        # exactly complementary to d1 as the assignment tries to equalize the
        # votes per sg
        self.assertEqual(d2, {0: 5,
                              1: 8,
                              2: 6,
                              3: 1,
                              4: 0})
        pl.append({'subject': 2, 'stimulusgroups': d2})

        d3a = ExperimentController._order(
            rounds_per_session=5,
            stimulusgroup_ids=list(range(10)),
            subject_id=2,  # repeated subject
            ordering_so_far=pl,
            prioritized=list(),
            random_seed=2,
            blocklist_stimulusgroup_ids=list(),
        )
        # exactly complementary to d2 as it's the same subject for a second
        # session and the assignment tries to avoid sg repetitions for the
        # subject
        self.assertEqual(d3a, {0: 9,
                               1: 2,
                               2: 7,
                               3: 4,
                               4: 3})

        # a different subject, the assignment should be random
        d3b = ExperimentController._order(
            rounds_per_session=5,
            stimulusgroup_ids=list(range(10)),
            subject_id=3,  # new subject
            ordering_so_far=pl,
            prioritized=list(),
            random_seed=2,
            blocklist_stimulusgroup_ids=list(),
        )
        self.assertEqual(d3b, {0: 2,
                               1: 0,
                               2: 5,
                               3: 4,
                               4: 3})

    def test_experiment_order_realistic(self):

        random.seed(0)

        prioritized = [
            {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},  # training
            {'session_idx': None, 'round_id': 5, 'stimulusgroup_id': 1},  # reliability
            {'session_idx': None, 'round_id': 10, 'stimulusgroup_id': 1},  # reliability
        ]
        blacklist_sgids = [0, 1]
        num_sg = 102  # training + reliability + 100 sgs
        rounds_per_session = 37  # 100/3 ~= 34, plus 1 training, 2 reliability
        subj_ids = list(range(1, 25)) * 3  # 24 votes per sg, need 3 people to scan through sgs
        random.shuffle(subj_ids)

        pl = list()
        ds = list()
        seed = 0
        for subj_id in subj_ids:
            d = ExperimentController._order(
                rounds_per_session=rounds_per_session,
                stimulusgroup_ids=list(range(num_sg)),
                subject_id=subj_id,
                ordering_so_far=pl,
                prioritized=prioritized,
                random_seed=seed,
                blocklist_stimulusgroup_ids=blacklist_sgids,
            )
            pl.append({'subject': subj_id, 'stimulusgroups': d})
            ds.append(d)
            seed += 1

        # ll is 2d array with (session_id, round_id) -> sg_id
        ll = list()
        for d in ds:
            ll.append([d[rid] for rid in sorted(d.keys())])
        ll = np.vstack(ll)

        # sg_id of 0th session evaluated:
        self.assertEqual(
            list(ll[0, :]),
            [0, 81, 16, 71, 18, 1, 42, 36, 68, 61, 1, 98, 20, 70, 7, 38, 77,
             86, 60, 57, 50, 43, 40, 84, 88, 99, 24, 30, 53, 56, 51, 15, 12,
             100, 21, 101, 66]
        )

        # sg_id of last session evaluated:
        self.assertEqual(
            list(ll[-1, :]),
            [0, 53, 42, 31, 17, 1, 28, 91, 96, 35, 1, 52, 20, 85, 10, 62, 93,
             60, 59, 99, 63, 68, 37, 41, 67, 74, 49, 66, 55, 21, 22, 14, 23,
             29, 72, 78, 69]
        )

        # plm is 2d array with (session_id, sg_id) -> count of sg tested
        plm = np.zeros([len(subj_ids), num_sg])
        for sess_id, (subj_id, d) in enumerate(zip(subj_ids, ds)):
            for rid, sgid in d.items():
                plm[sess_id, sgid] += 1

        # number of votes per sg:
        self.assertEqual(
            list(np.sum(plm, axis=0)),
            [72.0, 144.0, 24.0, 24.0, 24.0, 24.0, 24.0, 24.0, 24.0, 24.0, 25.0,
             24.0, 25.0, 25.0, 26.0, 24.0, 24.0, 25.0, 26.0, 25.0, 24.0, 25.0,
             24.0, 24.0, 24.0, 24.0, 25.0, 24.0, 24.0, 25.0, 24.0, 25.0, 25.0,
             24.0, 25.0, 25.0, 25.0, 24.0, 24.0, 24.0, 24.0, 24.0, 26.0, 24.0,
             24.0, 25.0, 24.0, 25.0, 24.0, 25.0, 24.0, 24.0, 25.0, 26.0, 24.0,
             25.0, 26.0, 24.0, 25.0, 25.0, 25.0, 24.0, 24.0, 25.0, 24.0, 24.0,
             25.0, 25.0, 25.0, 24.0, 26.0, 24.0, 25.0, 24.0, 24.0, 24.0, 24.0,
             24.0, 24.0, 24.0, 24.0, 24.0, 24.0, 24.0, 26.0, 25.0, 24.0, 26.0,
             24.0, 25.0, 25.0, 24.0, 24.0, 25.0, 24.0, 24.0, 25.0, 24.0, 24.0,
             25.0, 24.0, 24.0]

        )

        # number of votes per session:
        self.assertEqual(
            list(np.sum(plm, axis=1)),
            [37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37.,
             37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37.,
             37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37.,
             37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37.,
             37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37.,
             37., 37., 37., 37., 37., 37., 37.]
        )

        # number of times subject 1 (first subject) evaluated on each sg:
        self.assertEqual(
            list(np.sum(plm[indices(subj_ids, lambda x: x == 1), :], axis=0)),
            [3.0, 6.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
        )

        # number of times subject 2 evaluated on each sg:
        self.assertEqual(
            list(np.sum(plm[indices(subj_ids, lambda x: x == 2), :], axis=0)),
            [3.0, 6.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 2.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
        )

        # number of times subject 24 (last subject) evaluated on each sg:
        self.assertEqual(
            list(np.sum(plm[indices(subj_ids, lambda x: x == 24), :], axis=0)),
            [3.0, 6.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
        )

    def test_experiment_order_realistic_decoy_in_random_round(self):

        random.seed(0)

        prioritized = [
            {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},  # training
            {'session_idx': None, 'round_id': None, 'stimulusgroup_id': 1},  # reliability
        ]
        blacklist_sgids = [0, 1]
        num_sg = 102  # training + reliability + 100 sgs
        rounds_per_session = 37  # 100/3 ~= 34, plus 1 training, 2 reliability
        subj_ids = list(range(1, 25)) * 3  # 24 votes per sg, need 3 people to scan through sgs
        random.shuffle(subj_ids)

        pl = list()
        ds = list()
        seed = 0
        for subj_id in subj_ids:
            d = ExperimentController._order(
                rounds_per_session=rounds_per_session,
                stimulusgroup_ids=list(range(num_sg)),
                subject_id=subj_id,
                ordering_so_far=pl,
                prioritized=prioritized,
                random_seed=seed,
                blocklist_stimulusgroup_ids=blacklist_sgids,
            )
            pl.append({'subject': subj_id, 'stimulusgroups': d})
            ds.append(d)
            seed += 1

        # ll is 2d array with (session_id, round_id) -> sg_id
        ll = list()
        for d in ds:
            ll.append([d[rid] for rid in sorted(d.keys())])
        ll = np.vstack(ll)

        # sg_id of 0th session evaluated:
        self.assertEqual(
            list(ll[0, :]),
            [0, 36, 30, 101, 50, 7, 21, 93, 51, 15, 57, 38, 61, 81, 42, 77, 71,
             18, 53, 68, 70, 20, 98, 66, 40, 88, 16, 99, 86, 12, 24, 100, 1,
             56, 43, 60, 84]
        )

        # sg_id of last session evaluated:
        self.assertEqual(
            list(ll[-1, :]),
            [0, 84, 77, 80, 72, 64, 23, 15, 66, 44, 83, 13, 38, 82, 59, 3, 45,
             7, 91, 42, 97, 58, 98, 21, 1, 78, 16, 11, 94, 20, 25, 47, 68, 62,
             55, 96, 60]
        )

        # plm is 2d array with (session_id, sg_id) -> count of sg tested
        plm = np.zeros([len(subj_ids), num_sg])
        for sess_id, (subj_id, d) in enumerate(zip(subj_ids, ds)):
            for rid, sgid in d.items():
                plm[sess_id, sgid] += 1

        # number of votes per sg:
        self.assertEqual(
            list(np.sum(plm, axis=0)),
            [72.0, 72.0, 25.0, 25.0, 25.0, 25.0, 24.0, 25.0, 25.0, 25.0, 25.0,
             25.0, 25.0, 25.0, 25.0, 26.0, 25.0, 25.0, 25.0, 25.0, 26.0, 25.0,
             26.0, 25.0, 25.0, 26.0, 26.0, 25.0, 25.0, 26.0, 25.0, 27.0, 25.0,
             26.0, 25.0, 25.0, 26.0, 25.0, 25.0, 26.0, 25.0, 25.0, 26.0, 25.0,
             25.0, 25.0, 24.0, 26.0, 25.0, 25.0, 25.0, 25.0, 27.0, 25.0, 25.0,
             25.0, 25.0, 25.0, 25.0, 25.0, 25.0, 25.0, 25.0, 25.0, 25.0, 25.0,
             25.0, 25.0, 25.0, 25.0, 25.0, 26.0, 26.0, 25.0, 25.0, 25.0, 25.0,
             26.0, 25.0, 25.0, 25.0, 25.0, 25.0, 25.0, 26.0, 25.0, 25.0, 24.0,
             25.0, 27.0, 25.0, 25.0, 25.0, 26.0, 25.0, 25.0, 25.0, 25.0, 26.0,
             25.0, 25.0, 25.0]
        )

        # number of votes per session:
        self.assertEqual(
            list(np.sum(plm, axis=1)),
            [37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37.,
             37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37.,
             37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37.,
             37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37.,
             37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37., 37.,
             37., 37., 37., 37., 37., 37., 37.]
        )

        # number of times subject 1 (first subject) evaluated on each sg:
        self.assertEqual(
            list(np.sum(plm[indices(subj_ids, lambda x: x == 1), :], axis=0)),
            [3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0,
             1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
             1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
        )


class TestExperimentController(TestCase):

    @staticmethod
    def _populate_stimulusgroups(ec: ExperimentController):
        sg: dict
        for sg in ec.experiment_config.stimulus_config.stimulusgroups:
            StimulusGroup.objects.create(
                experiment=ec.experiment,
                stimulusgroup_id=sg['stimulusgroup_id'])

    def test_exp_order(self):
        e = Experiment(
            title='Zhi ACR',
            description="Zhi's ACR experiment",
        )
        e.save()
        scfg = StimulusConfig({
            "contents": [
                {
                    "content_id": 0,
                    "name": "CTS3E1_B__15_55_16_0",
                }
            ],
            "stimuli": [
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4",  # noqa E501
                    "stimulus_id": 0,
                    "type": "video/mp4",
                    "content_id": 0,
                },
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4",  # noqa E501
                    "stimulus_id": 1,
                    "type": "video/mp4",
                    "content_id": 0,
                }
            ],
            "stimulusvotegroups": [
                {
                    "stimulus_ids": [
                        0
                    ],
                    "stimulusvotegroup_id": 0
                },
                {
                    "stimulus_ids": [
                        1
                    ],
                    "stimulusvotegroup_id": 1
                }
            ],
            'stimulusgroups': [
                {'info': {'flavors': ['training']}, 'stimulusgroup_id': 0, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 4, 'stimulusvotegroup_ids': [1]},
                {'stimulusgroup_id': 2, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 3, 'stimulusvotegroup_ids': [0]},
                {'info': {'flavors': ['decoy']}, 'stimulusgroup_id': 5, 'stimulusvotegroup_ids': [0]},
            ],
        })
        pcfg = ExperimentConfig(
            stimulus_config=scfg,
            config={
                'title': 'test',
                'description': 'test',
                "methodology": "acr",
                'vote_scale': 'FIVE_POINT',
                'rounds_per_session': 8,
                'random_seed': 3,
                'prioritized': [
                    {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},
                    {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 5},
                    {'session_idx': 1, 'round_id': 5, 'stimulusgroup_id': 5},
                    {'session_idx': 2, 'round_id': 5, 'stimulusgroup_id': 5},
                ]
            })
        ec = ExperimentController(experiment=e, experiment_config=pcfg)
        self.assertEqual(ec.experiment.title, 'Zhi ACR')

        stimulusgroups = ec.get_and_assert_current_stimulusgroups()
        self.assertEqual(len(stimulusgroups), 0)

        self._populate_stimulusgroups(ec)

        s = Subject()
        s.save()
        sess = ec.add_session(s)
        stimulusgroups = ec.get_and_assert_current_stimulusgroups()
        self.assertEqual(stimulusgroups[0].round_set.count(), 2)
        self.assertEqual(stimulusgroups[1].round_set.count(), 2)
        self.assertEqual(stimulusgroups[2].round_set.count(), 1)
        self.assertEqual(stimulusgroups[3].round_set.count(), 1)
        self.assertEqual(stimulusgroups[4].round_set.count(), 2)
        self.assertEqual(sess.subject, s)

        s2 = Subject()
        s2.save()
        ec.add_session(s2)
        self.assertEqual(stimulusgroups[0].round_set.count(), 4)
        self.assertEqual(stimulusgroups[1].round_set.count(), 3)
        self.assertEqual(stimulusgroups[2].round_set.count(), 3)
        self.assertEqual(stimulusgroups[3].round_set.count(), 2)
        self.assertEqual(stimulusgroups[4].round_set.count(), 4)

        sinfo = ec.get_session_info(sess)
        self.assertEqual(
            sinfo,
            {'session_id': 1, 'subject': '1',
             'rounds': [
                 {'round_id': 0, 'stimulusgroup_id': 0, 'stimulusvotegroups': []},
                 {'round_id': 5, 'stimulusgroup_id': 5, 'stimulusvotegroups': []},
                 {'round_id': 1, 'stimulusgroup_id': 0, 'stimulusvotegroups': []},
                 {'round_id': 2, 'stimulusgroup_id': 4, 'stimulusvotegroups': []},
                 {'round_id': 3, 'stimulusgroup_id': 3, 'stimulusvotegroups': []},
                 {'round_id': 4, 'stimulusgroup_id': 2, 'stimulusvotegroups': []},
                 {'round_id': 6, 'stimulusgroup_id': 5, 'stimulusvotegroups': []},
                 {'round_id': 7, 'stimulusgroup_id': 4, 'stimulusvotegroups': []}]
             })

        stiminfo = ec.get_stimuli_info()
        self.assertEqual(
            stiminfo,
            {'contents': [{'content_id': 0, 'name': 'CTS3E1_B__15_55_16_0'}],
             'stimuli': [{'path': '/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4', 'stimulus_id': 0, 'type': 'video/mp4', 'content_id': 0},  # noqa E501
                         {'path': '/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4', 'stimulus_id': 1, 'type': 'video/mp4', 'content_id': 0}],  # noqa E501
             'stimulusvotegroups': [{'stimulus_ids': [0], 'stimulusvotegroup_id': 0},
                                    {'stimulus_ids': [1], 'stimulusvotegroup_id': 1}],
             'stimulusgroups': [{'info': {'flavors': ['training']}, 'stimulusgroup_id': 0, 'stimulusvotegroup_ids': [0]},
                                {'stimulusgroup_id': 4, 'stimulusvotegroup_ids': [1]},
                                {'stimulusgroup_id': 2, 'stimulusvotegroup_ids': [0]},
                                {'stimulusgroup_id': 3, 'stimulusvotegroup_ids': [0]},
                                {'info': {'flavors': ['decoy']}, 'stimulusgroup_id': 5, 'stimulusvotegroup_ids': [0]}]}
        )

    def test_exp_order_realistic(self):

        random_seed = 0

        random.seed(random_seed)
        prioritized = [
            {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},  # training
            {'session_idx': None, 'round_id': 5, 'stimulusgroup_id': 1},  # reliability
            {'session_idx': None, 'round_id': 10, 'stimulusgroup_id': 1},  # reliability
        ]
        blacklist_sgids = [0, 1]
        num_sg = 42  # training + reliability + 40 sgs
        rounds_per_session = 30
        subj_ids = list(range(1, 21)) * 2
        random.shuffle(subj_ids)

        # (30 - 3) = 27 votes / session
        # 20 * 2 = 40 sessions
        # total number of votes = 27 * 40 = 1080
        # total number of test sgs = 40
        # average votes per sg = 1080 / 40 = 27

        for sid in range(1, 21):
            s = Subject()
            s.save()

        e = Experiment(
            title='Zhi ACR 2',
            description="Zhi's ACR experiment",
        )
        e.save()

        scfg = StimulusConfig({
            "contents": [
                {
                    "content_id": 0,
                    "name": "CTS3E1_B__15_55_16_0",
                }
            ],
            "stimuli": [
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4",  # noqa E501
                    "stimulus_id": 0,
                    "type": "video/mp4",
                    "content_id": 0,
                },
            ],
            "stimulusvotegroups": [
                {
                    "stimulus_ids": [
                        0
                    ],
                    "stimulusvotegroup_id": 0
                },
            ],
            'stimulusgroups': [{'stimulusgroup_id': sgid,
                                'stimulusvotegroup_ids': [0]}
                               for sgid in range(num_sg)]
        })
        pcfg = ExperimentConfig(
            stimulus_config=scfg,
            config={
                'title': 'test',
                'description': 'test',
                "methodology": "acr",
                'vote_scale': 'FIVE_POINT',
                'rounds_per_session': rounds_per_session,
                'random_seed': random_seed,
                'prioritized': prioritized,
                'blocklist_stimulusgroup_ids': blacklist_sgids,
            })
        ec = ExperimentController(experiment=e, experiment_config=pcfg)

        self._populate_stimulusgroups(ec)

        for session_id, subj_id in enumerate(subj_ids):
            # print(f"add session {session_id} for subject {subj_id}")
            s = Subject.objects.get(id=subj_id)
            ec.add_session(s)

        stimulusgroups = ec.get_and_assert_current_stimulusgroups()
        self.assertEqual(len(stimulusgroups), 42)

        self.assertEqual(StimulusGroup.objects.get(stimulusgroup_id=0).round_set.count(), 40)
        self.assertEqual(StimulusGroup.objects.get(stimulusgroup_id=1).round_set.count(), 80)
        self.assertEqual(StimulusGroup.objects.get(stimulusgroup_id=2).round_set.count(), 27)
        self.assertEqual(StimulusGroup.objects.get(stimulusgroup_id=3).round_set.count(), 27)
        self.assertEqual(StimulusGroup.objects.get(stimulusgroup_id=40).round_set.count(), 27)
        self.assertEqual(StimulusGroup.objects.get(stimulusgroup_id=41).round_set.count(), 26)

        # test on ExperimentController._populate_stimulus_info():
        # retrieve sg instead of creating
        eo2 = ExperimentController(experiment=e, experiment_config=pcfg)

        stimulusgroups = eo2.get_and_assert_current_stimulusgroups()
        self.assertEqual(len(stimulusgroups), 42)

    def test_session_status(self):
        e = Experiment(
            title='Zhi ACR',
            description="Zhi's ACR experiment",
        )
        e.save()
        scfg = StimulusConfig({
            "contents": [
                {
                    "content_id": 0,
                    "name": "CTS3E1_B__15_55_16_0",
                }
            ],
            "stimuli": [
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4",  # noqa E501
                    "stimulus_id": 0,
                    "type": "video/mp4",
                    "content_id": 0,
                },
                {
                    "path": "/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4",  # noqa E501
                    "stimulus_id": 1,
                    "type": "video/mp4",
                    "content_id": 0,
                }
            ],
            "stimulusvotegroups": [
                {
                    "stimulus_ids": [
                        0
                    ],
                    "stimulusvotegroup_id": 0
                },
                {
                    "stimulus_ids": [
                        1
                    ],
                    "stimulusvotegroup_id": 1
                }
            ],
            'stimulusgroups': [
                {'info': {'flavors': ['training']}, 'stimulusgroup_id': 0, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 4, 'stimulusvotegroup_ids': [1]},
                {'stimulusgroup_id': 2, 'stimulusvotegroup_ids': [0]},
                {'stimulusgroup_id': 3, 'stimulusvotegroup_ids': [0]},
                {'info': {'flavors': ['decoy']}, 'stimulusgroup_id': 5, 'stimulusvotegroup_ids': [0]},
            ],
        })
        pcfg = ExperimentConfig(
            stimulus_config=scfg,
            config={
                'title': 'test',
                'description': 'test',
                "methodology": "acr",
                'vote_scale': 'FIVE_POINT',
                'rounds_per_session': 8,
                'random_seed': 3,
                'prioritized': [
                    {'session_idx': None, 'round_id': 0, 'stimulusgroup_id': 0},
                    {'session_idx': 0, 'round_id': 5, 'stimulusgroup_id': 5},
                    {'session_idx': 1, 'round_id': 5, 'stimulusgroup_id': 5},
                    {'session_idx': 2, 'round_id': 5, 'stimulusgroup_id': 5},
                ]
            })
        ec = ExperimentController(experiment=e, experiment_config=pcfg)

        self._populate_stimulusgroups(ec)

        subj = Subject()
        subj.save()
        sess = Session(experiment=e, subject=subj)  # limbo session
        # self.assertEqual(ec.get_session_status(sess), SessionStatus.UNINITIALIZED)  # (6/13/2023: no longer the case)
        with self.assertRaises(ValueError):
            ec.get_session_status(sess)

        subj2 = Subject()
        subj2.save()
        ec.add_session(subj2)
        sess2 = e.session_set.get(subject=subj2)
        self.assertEqual(ec.get_session_status(sess2), SessionStatus.INITIALIZED)


class TestInsertAdditionsIntoSteps(TestCase):

    def test_1(self):
        steps = [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ]
        addition = \
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "z"}}
        ExperimentController._insert_addition_into_steps(addition, steps)
        self.assertEqual(steps, [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "z"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ])

    def test_2(self):
        steps = [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "a"}},
            {'position': {'round_id': 1, 'before_or_after': 'before'}, 'context': {'title': "b"}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 1, 'before_or_after': 'after'}, 'context': {'title': "w"}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ]
        addition = \
            {'position': {'round_id': 1, 'before_or_after': 'before'}, 'context': {'title': "c"}}
        ExperimentController._insert_addition_into_steps(addition, steps)
        self.assertEqual(steps, [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "a"}},
            {'position': {'round_id': 1, 'before_or_after': 'before'}, 'context': {'title': "b"}},
            {'position': {'round_id': 1, 'before_or_after': 'before'}, 'context': {'title': "c"}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 1, 'before_or_after': 'after'}, 'context': {'title': "w"}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ])

    def test_3(self):
        steps = [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "a"}},
            {'position': {'round_id': 1, 'before_or_after': 'before'}, 'context': {'title': "b"}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ]
        addition = \
            {'position': {'round_id': 5, 'before_or_after': 'before'}, 'context': {'title': "c"}}
        with self.assertRaises(AssertionError):
            ExperimentController._insert_addition_into_steps(addition, steps)

    def test_3a(self):
        steps = [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ]
        addition = \
            {'position': {'round_id': -1, 'before_or_after': 'before'}, 'context': {'title': "z"}}
        with self.assertRaises(AssertionError):
            ExperimentController._insert_addition_into_steps(addition, steps)

    def test_4(self):
        steps = [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ]
        addition = \
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "a"}}
        ExperimentController._insert_addition_into_steps(addition, steps)
        self.assertEqual(steps, [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "a"}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ])

    def test_5(self):
        steps = [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "a"}},
            {'position': {'round_id': 1, 'before_or_after': 'before'}, 'context': {'title': "b"}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 1, 'before_or_after': 'after'}, 'context': {'title': "w"}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
            {'position': {'round_id': 4, 'before_or_after': 'after'}, 'context': {'title': "q"}},
        ]
        addition = \
            {'position': {'round_id': 4, 'before_or_after': 'after'}, 'context': {'title': "c"}}
        ExperimentController._insert_addition_into_steps(addition, steps)
        self.assertEqual(steps, [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "a"}},
            {'position': {'round_id': 1, 'before_or_after': 'before'}, 'context': {'title': "b"}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 1, 'before_or_after': 'after'}, 'context': {'title': "w"}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
            {'position': {'round_id': 4, 'before_or_after': 'after'}, 'context': {'title': "q"}},
            {'position': {'round_id': 4, 'before_or_after': 'after'}, 'context': {'title': "c"}},
        ])

    def test_6(self):
        steps = [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "a"}},
            {'position': {'round_id': 1, 'before_or_after': 'before'}, 'context': {'title': "b"}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 1, 'before_or_after': 'after'}, 'context': {'title': "w"}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ]
        addition = \
            {'position': {'round_id': 5, 'before_or_after': 'after'}, 'context': {'title': "c"}}
        with self.assertRaises(AssertionError):
            ExperimentController._insert_addition_into_steps(addition, steps)

    def test_7(self):
        steps = [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "x"}},
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "y"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "a"}},
            {'position': {'round_id': 1, 'before_or_after': 'before'}, 'context': {'title': "b"}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 1, 'before_or_after': 'after'}, 'context': {'title': "w"}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ]
        addition = \
            {'position': {'round_id': -1, 'before_or_after': 'after'}, 'context': {'title': "c"}}
        with self.assertRaises(AssertionError):
            ExperimentController._insert_addition_into_steps(addition, steps)

    def test_8(self):
        steps = [
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ]
        addition = \
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "z"}}
        ExperimentController._insert_addition_into_steps(addition, steps)
        self.assertEqual(steps, [
            {'position': {'round_id': 0, 'before_or_after': 'before'}, 'context': {'title': "z"}},
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ])

    def test_9(self):
        steps = [
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ]
        addition = \
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}}
        ExperimentController._insert_addition_into_steps(addition, steps)
        self.assertEqual(steps, [
            {'position': {'round_id': 0}, 'content': {'stimulusgroup_id': 0}},
            {'position': {'round_id': 0, 'before_or_after': 'after'}, 'context': {'title': "z"}},
            {'position': {'round_id': 1}, 'content': {'stimulusgroup_id': 1}},
            {'position': {'round_id': 2}, 'content': {'stimulusgroup_id': 2}},
            {'position': {'round_id': 3}, 'content': {'stimulusgroup_id': 3}},
            {'position': {'round_id': 4}, 'content': {'stimulusgroup_id': 4}},
        ])


class TestExperimentControllerWithConfig(TestCase):

    def test_ec_with_config(self):
        config_filepath = NestConfig.tests_resource_path('cvxhull_subjexp_toy_x.json')
        with open(config_filepath, 'rt') as fp:
            config = json.load(fp)
        scfg = StimulusConfig(config['stimulus_config'])
        ecfg = ExperimentConfig(stimulus_config=scfg,
                                config=config['experiment_config'])
        e = Experiment(
            title='Zhi ACR',
            description="Zhi's ACR experiment",
        )
        e.save()
        ec = ExperimentController(experiment=e, experiment_config=ecfg)
        ec.populate_stimuli()

        subj = Subject.objects.create()
        ec.add_session(subj)
        sess = e.session_set.get(subject=subj)
        self.assertEqual(ec.get_session_status(sess), SessionStatus.INITIALIZED)
        cs = Content.objects.all()
        sgs = StimulusGroup.objects.all()
        svgs = StimulusVoteGroup.objects.all()
        stims = Stimulus.objects.all()
        self.assertEqual(len(cs), 1)
        self.assertEqual(cs[0].content_id, 0)
        self.assertEqual(stims[0].content, cs[0])
        self.assertEqual(stims[1].content, cs[0])
        self.assertEqual(svgs[0].stimulusvotegroup_id, 0)
        self.assertEqual(svgs[1].stimulusvotegroup_id, 1)
        self.assertEqual(StimulusVoteGroup.find_stimulusvotegroups_from_stimulus(stims[0]), [svgs[0]])
        self.assertEqual(StimulusVoteGroup.find_stimulusvotegroups_from_stimulus(stims[1]), [svgs[1]])
        self.assertEqual(svgs[0].stimulusgroup, sgs[0])
        self.assertEqual(svgs[1].stimulusgroup, sgs[1])

        self.assertEqual(Round.objects.count(), 2)
        ec.delete_session(sess.id)
        self.assertEqual(Round.objects.count(), 0)

nest_site/nest/tests/control_tests.py
Netflix/e2nest
import glob
import json
import os
import random
import shutil

from django.contrib.auth.models import User
from django.test import TestCase
from nest.config import ExperimentConfig, NestConfig, StimulusConfig
from nest.control import ExperimentController, SessionStatus
from nest.helpers import import_python_file
from nest.io import ESUtilities, ExperimentUtils, \
    export_sureal_dataset, import_sureal_dataset, \
    SurealPairedCompDatasetReaderPlus, SurealRawDatasetReaderPlus, UserUtils
from nest.io_utils import ExperimentConfigFileUtils
from nest.models import Condition, Content, Experiment, Experimenter, \
    ExperimentRegister, FivePointVote, Round, Session, Stimulus, \
    StimulusGroup, StimulusVoteGroup, Subject, TafcVote, Vote, VoteRegister, \
    Zero2HundredVote
from nest.sites import NestSite
from nest_site.settings import map_media_local_to_url


class TestDatasetReader(TestCase):

    def test_populate_nest_nflx_public(self):
        dataset_path = NestConfig.tests_resource_path(
            'NFLX_dataset_public_raw_last4outliers_tiny.py')
        scale = 'FIVE_POINT'
        dataset = import_python_file(dataset_path)
        import_sureal_dataset(dataset, scale)

        self.assertEqual(Experiment.objects.count(), 1)
        self.assertEqual(Content.objects.count(), 3)
        self.assertEqual(Subject.objects.count(), 30)
        self.assertEqual(Session.objects.count(), 30)
        self.assertEqual(Stimulus.objects.count(), 28)
        self.assertEqual(Round.objects.count(), 840)
        self.assertEqual(Vote.objects.count(), 840)
        self.assertEqual(FivePointVote.objects.count(), 840)
        self.assertEqual(VoteRegister.objects.count(), 28)
        svg = StimulusVoteGroup.objects.get(id=1)
        self.assertEqual(VoteRegister.objects.get(stimulusvotegroup=svg).stimulus_order, 1)
        self.assertEqual(StimulusGroup.objects.get(id=1).round_set.count(), 30)
        self.assertEqual(StimulusGroup.objects.get(id=1).stimulusgroup_id, 9)

        reader = SurealRawDatasetReaderPlus(dataset)
        self.assertEqual(len(reader._retrieve_or_create_stimuli()), 28)
        sd = reader.get_stimulus_dict()
        self.assertEqual(len(sd['stimulusgroups']), 28)
        self.assertEqual(sd['stimulusgroups'][0]['stimulusgroup_id'], 9)
        self.assertEqual(sd['stimulusgroups'][10]['stimulusgroup_id'], 0)
        self.assertEqual(sd['stimulusgroups'][20]['stimulusgroup_id'], 27)

        self.assertEqual(Round.objects.get(id=1).round_id, 0)
        self.assertEqual(Round.objects.get(id=150).round_id, 4)
        self.assertEqual(Round.objects.get(id=250).round_id, 8)

    def test_populate_nest_os_style_dict(self):
        dataset_path = NestConfig.tests_resource_path(
            'vmafplusstudy_laptop_raw_tiny.py')
        scale = '0_TO_100'
        dataset = import_python_file(dataset_path)
        import_sureal_dataset(dataset, scale)

        self.assertEqual(Experiment.objects.count(), 1)
        self.assertEqual(Content.objects.count(), 3)
        self.assertEqual(Subject.objects.count(), 41)
        self.assertEqual(Session.objects.count(), 41)
        self.assertEqual(Stimulus.objects.count(), 32)
        self.assertEqual(Round.objects.count(), 1312)
        self.assertEqual(Vote.objects.count(), 653)
        self.assertEqual(Zero2HundredVote.objects.count(), 653)
        self.assertEqual(VoteRegister.objects.count(), 32)
        self.assertEqual(StimulusGroup.objects.get(id=1).round_set.count(), 41)
        self.assertEqual(StimulusGroup.objects.get(id=1).stimulusgroup_id, 1000)

        reader = SurealRawDatasetReaderPlus(dataset)
        self.assertEqual(len(reader._retrieve_or_create_stimuli()), 32)
        sd = reader.get_stimulus_dict()
        self.assertEqual(len(sd['stimulusgroups']), 32)
        self.assertEqual(sd['stimulusgroups'][0]['stimulusgroup_id'], 1000)
        self.assertEqual(sd['stimulusgroups'][10]['stimulusgroup_id'], 17)
        self.assertEqual(sd['stimulusgroups'][20]['stimulusgroup_id'], 34)

        self.assertEqual(Round.objects.get(id=1).round_id, 0)
        self.assertEqual(Round.objects.get(id=150).round_id, 3)
        self.assertEqual(Round.objects.get(id=250).round_id, 6)

    def test_populate_nest_mismatched_vote_class(self):
        dataset_path = NestConfig.tests_resource_path(
            'vmafplusstudy_laptop_raw_tiny.py')
        scale = 'FIVE_POINT'
        dataset = import_python_file(dataset_path)
        with self.assertRaises(AssertionError):
            import_sureal_dataset(dataset, scale)

    def test_populate_nest_pc(self):
        dataset_path = NestConfig.tests_resource_path(
            'lukas_pc_dataset_tiny.py')
        scale = '2AFC'
        dataset = import_python_file(dataset_path)
        import_sureal_dataset(dataset, scale)

        self.assertEqual(Experiment.objects.count(), 1)
        self.assertEqual(Content.objects.count(), 2)
        self.assertEqual(Subject.objects.count(), 15)
        self.assertEqual(Session.objects.count(), 15)
        self.assertEqual(Stimulus.objects.count(), 16)
        self.assertEqual(Round.objects.count(), 840)
        self.assertEqual(Vote.objects.count(), 840)
        self.assertEqual(TafcVote.objects.count(), 840)
        self.assertEqual(VoteRegister.objects.count(), 206)
        svg = StimulusVoteGroup.objects.get(id=1)
        self.assertEqual(VoteRegister.objects.
                         filter(stimulusvotegroup=svg)[0].stimulus_order, 1)
        self.assertEqual(VoteRegister.objects.
                         filter(stimulusvotegroup=svg)[1].stimulus_order, 2)
        self.assertEqual(StimulusGroup.objects.get(id=1).round_set.count(), 2)
        self.assertEqual(StimulusGroup.objects.get(id=2).round_set.count(), 12)
        self.assertEqual(StimulusGroup.objects.get(id=3).round_set.count(), 8)
        self.assertEqual(StimulusGroup.objects.get(id=4).round_set.count(), 12)
        self.assertEqual(StimulusGroup.objects.get(id=5).round_set.count(), 13)
        self.assertEqual(StimulusGroup.objects.get(id=1).stimulusgroup_id, 0)
        self.assertEqual(StimulusGroup.objects.get(id=2).stimulusgroup_id, 1)
        self.assertEqual(StimulusGroup.objects.get(id=3).stimulusgroup_id, 2)
        self.assertEqual(StimulusGroup.objects.get(id=4).stimulusgroup_id, 3)
        self.assertEqual(StimulusGroup.objects.get(id=5).stimulusgroup_id, 4)

        reader = SurealPairedCompDatasetReaderPlus(dataset)
        self.assertEqual(len(reader._retrieve_or_create_stimuli()), 16)
        with self.assertRaises(NotImplementedError):
            reader.get_stimulusgroups()

    def test_populate_nest_linear_repeat(self):
        dataset_path = NestConfig.tests_resource_path(
            'NFLX_dataset_public_raw_last4outliers_tiny.py')
        scale = 'FIVE_POINT'
        dataset = import_python_file(dataset_path)
        import_sureal_dataset(dataset, scale)
        with self.assertRaises(AssertionError):
            import_sureal_dataset(dataset, scale)

    def test_populate_nest_pc_repeat(self):
        dataset_path = NestConfig.tests_resource_path(
            'lukas_pc_dataset_tiny.py')
        scale = '2AFC'
        dataset = import_python_file(dataset_path)
        import_sureal_dataset(dataset, scale)
        with self.assertRaises(AssertionError):
            import_sureal_dataset(dataset, scale)

    def test_export_dataset_single(self):
        dataset_path = NestConfig.tests_resource_path(
            'NFLX_dataset_public_raw_last4outliers_tiny.py')
        import_sureal_dataset(import_python_file(dataset_path), 'FIVE_POINT')
        dataset = export_sureal_dataset('NFLX_public')
        self.assertEqual(dataset.dataset_name, 'NFLX_public')
        self.assertEqual(len(dataset.ref_videos), 3)
        self.assertEqual(len(dataset.dis_videos), 28)
        self.assertEqual(len(dataset.dis_videos[0]['os']), 30)
        self.assertEqual(len(dataset.dis_videos[1]['os']), 30)
        self.assertEqual(len(dataset.dis_videos[2]['os']), 30)
        self.assertEqual(list(dataset.dis_videos[2]['os'].keys())[0], 'NFLX_public-#0')

    def test_export_dataset_single2(self):
        dataset_path = NestConfig.tests_resource_path(
            'vmafplusstudy_laptop_raw_tiny.py')
        import_sureal_dataset(import_python_file(dataset_path), '0_TO_100')
        dataset = export_sureal_dataset('VMAF_Plus_DB')
        self.assertEqual(dataset.dataset_name, 'VMAF_Plus_DB')
        self.assertEqual(len(dataset.ref_videos), 3)
        self.assertEqual(len(dataset.dis_videos), 32)
        self.assertEqual(len(dataset.dis_videos[0]['os']), 19)
        self.assertEqual(len(dataset.dis_videos[1]['os']), 19)
        self.assertEqual(len(dataset.dis_videos[2]['os']), 19)
        self.assertEqual(list(dataset.dis_videos[2]['os'].keys())[0], 'VMAF_Plus_DB-aschuler')

    def test_export_dataset_double(self):
        dataset_path = NestConfig.tests_resource_path(
            'lukas_pc_dataset_tiny.py')
        dataset = import_python_file(dataset_path)
        dataset.dataset_name = 'Lukas_SPIE14_tiny'
        import_sureal_dataset(dataset, '2AFC')
        dataset = export_sureal_dataset('Lukas_SPIE14_tiny')
        self.assertEqual(dataset.dataset_name, 'Lukas_SPIE14_tiny')
        self.assertEqual(len(dataset.ref_videos), 2)
        self.assertEqual(len(dataset.dis_videos), 16)
        self.assertEqual(len(dataset.dis_videos[0]['os']), 65)
        self.assertEqual(len(dataset.dis_videos[1]['os']), 86)
        self.assertEqual(len(dataset.dis_videos[2]['os']), 82)
        self.assertEqual(len(dataset.dis_videos[3]['os']), 61)
        self.assertEqual(len(dataset.dis_videos[4]['os']), 54)
        self.assertEqual(len(dataset.dis_videos[14]['os']), 22)
        self.assertEqual(len(dataset.dis_videos[15]['os']), 8)
        self.assertEqual(list(dataset.dis_videos[2]['os'].keys())[0], ('Lukas_SPIE14_tiny-David R.', 4))

    def test_export_dataset_double_ignore_against(self):
        # meant for DCR but could only find dataset of PC to create the test
        dataset_path = NestConfig.tests_resource_path(
            'lukas_pc_dataset_tiny.py')
        dataset = import_python_file(dataset_path)
        dataset.dataset_name = 'Lukas_SPIE14_tiny'
        import_sureal_dataset(dataset, '2AFC')
        dataset = export_sureal_dataset('Lukas_SPIE14_tiny', ignore_against=True)
        self.assertEqual(dataset.dataset_name, 'Lukas_SPIE14_tiny')
        self.assertEqual(len(dataset.ref_videos), 2)
        self.assertEqual(len(dataset.dis_videos), 16)
        self.assertEqual(len(dataset.dis_videos[0]['os']), 14)
        self.assertEqual(len(dataset.dis_videos[1]['os']), 15)
        self.assertEqual(len(dataset.dis_videos[2]['os']), 15)
        self.assertEqual(len(dataset.dis_videos[3]['os']), 15)
        self.assertEqual(len(dataset.dis_videos[4]['os']), 15)
        self.assertEqual(len(dataset.dis_videos[5]['os']), 15)
        self.assertEqual(len(dataset.dis_videos[15]['os']), 5)
        self.assertEqual(dataset.dis_videos[0]['os']['Lukas_SPIE14_tiny-David R.'], [1.0, 1.0, 1.0, 1.0, 1.0, 1.0])
        self.assertEqual(dataset.dis_videos[0]['os']['Lukas_SPIE14_tiny-dingo'], [1.0, 1.0, 1.0, 1.0])


class TestExperimentController(TestCase):

    @staticmethod
    def create_config_from_sureal_dataset(dataset, seed):
        try:
            dataset_reader = SurealPairedCompDatasetReaderPlus(dataset)
        except AssertionError:
            dataset_reader = SurealRawDatasetReaderPlus(dataset)
        sd = dataset_reader.get_stimulus_dict()
        return dataset_reader.make_experiment_config_dict(
            sd['contents'],
            sd['stimuli'],
            sd['stimulusvotegroups'],
            sd['stimulusgroups'],
            seed)

    def test_populate_nest_nflx_public(self):
        dataset_path = NestConfig.tests_resource_path(
            'NFLX_dataset_public_raw_last4outliers_tiny.py')
        scale = 'FIVE_POINT'
        dataset = import_python_file(dataset_path)
        import_sureal_dataset(dataset, scale)

        config = self.create_config_from_sureal_dataset(dataset, seed=1)
        scfg = StimulusConfig(config['stimulus_config'], skip_path_check=True)
        ecfg = ExperimentConfig(stimulus_config=scfg,
                                config=config['experiment_config'])
        exp = Experiment.objects.get(title=dataset.dataset_name)
        ec = ExperimentController(experiment=exp, experiment_config=ecfg)

        sess = exp.session_set.first()
        self.assertEqual(ec.get_session_status(sess), SessionStatus.FINISHED)

        sinfo: dict = ec.get_session_info(sess)
        self.assertEqual(sinfo['subject'], 'NFLX_public-#0')
        self.assertEqual(sinfo['session_id'], 1)
        self.assertEqual(sinfo['rounds'][-1]['round_id'], 27)
        self.assertEqual(sinfo['rounds'][-1]['stimulusgroup_id'], 2)
        self.assertEqual(sinfo['rounds'][-1]['stimulusvotegroups'][0]['stimulusvotegroup_id'], 2)
        self.assertEqual(sinfo['rounds'][-1]['stimulusvotegroups'][0]['vote'], 5.0)

        einfo: dict = ec.get_experiment_info()
        self.assertEqual(einfo['title'], 'NFLX_public')
        self.assertEqual(einfo['description'], None)
        self.assertEqual(len(einfo['sessions']), 30)
        self.assertEqual(len(einfo['sessions'][0]['rounds']), 28)
        self.assertEqual(len(einfo['stimuli_info']['contents']), 3)
        self.assertEqual(len(einfo['stimuli_info']['stimuli']), 28)
        self.assertEqual(len(einfo['stimuli_info']['stimulusvotegroups']), 28)
        self.assertEqual(len(einfo['stimuli_info']['stimulusgroups']), 28)

        df = ExperimentController.denormalize_experiment_info(einfo)
        self.assertEqual(df.shape, (840, 8))
        self.assertEqual(df.iloc[0].subject, 'NFLX_public-#0')
        self.assertEqual(df.iloc[0].session_id, 1)
        self.assertEqual(df.iloc[0].round_id, 0)
        self.assertEqual(df.iloc[0].sg_id, 9)
        self.assertEqual(df.iloc[0].svg_id, 9)
        self.assertEqual(df.iloc[0].vote, 1.0)
        self.assertEqual(df.iloc[0].sid, 9)
        self.assertEqual(df.iloc[0].path, '[path to dataset videos]/dis/BigBuckBunny_20_288_375.yuv')

        sess.round_set.first().vote_set.first().delete()
        self.assertEqual(ec.get_session_status(sess), SessionStatus.PARTIALLY_FINISHED)

        sess2 = ExperimentUtils.add_session_to_experiment(
            exp.title, 'zli', config=config, skip_path_check=True)
        self.assertEqual(sess2.round_set.count(), 28)
        self.assertEqual(sess2.subject.user.username, 'zli')

        ExperimentUtils.delete_session_by_id(
            sess2.id, exp.title, config=config, skip_path_check=True)
        with self.assertRaises(Session.DoesNotExist):
            Session.objects.get(id=sess2.id)

        subj = Subject()
        subj.save()
        ec.add_session(subj)
        sess2 = exp.session_set.get(subject=subj)
        self.assertEqual(ec.get_session_status(sess2), SessionStatus.INITIALIZED)

        ExperimentUtils.update_subject_for_session(
            exp.title, exp.session_set.all()[3].id,
            'zli', config=config, skip_path_check=True)
        sess_3rd = exp.session_set.all()[3]
        self.assertEqual(sess_3rd.subject.user.username, 'zli')
        self.assertEqual(ec.get_session_status(sess_3rd), SessionStatus.FINISHED)

        ExperimentUtils.reset_unfinished_session(exp.title, sess.id,
                                                 config=config,
                                                 skip_path_check=True)
        self.assertEqual(ec.get_session_status(sess), SessionStatus.INITIALIZED)

        steps = ec.get_session_steps(sess)
        self.assertEqual(len(steps), 29)
        self.assertEqual(steps[1], {'context': {'stimulusgroup_id': 9}, 'position': {'round_id': 0}})
        self.assertEqual(steps[-1], {'context': {'stimulusgroup_id': 2}, 'position': {'round_id': 27}})

        # test delete session
        self.assertEqual(Session.objects.count(), 31)
        self.assertEqual(Round.objects.count(), 868)
        self.assertEqual(Vote.objects.count(), 812)
        for sess in exp.session_set.all():
            ec.delete_session(sess.id)
        self.assertEqual(Session.objects.count(), 0)
        self.assertEqual(Round.objects.count(), 0)
        self.assertEqual(Vote.objects.count(), 0)

    def test_populate_nest_os_style_dict(self):
        dataset_path = NestConfig.tests_resource_path(
            'vmafplusstudy_laptop_raw_tiny.py')
        scale = '0_TO_100'
        dataset = import_python_file(dataset_path)
        import_sureal_dataset(dataset, scale)

        config = self.create_config_from_sureal_dataset(dataset, seed=1)
        scfg = StimulusConfig(config['stimulus_config'], skip_path_check=True)
        ecfg = ExperimentConfig(stimulus_config=scfg,
                                config=config['experiment_config'])
        exp = Experiment.objects.get(title=dataset.dataset_name)
        ec = ExperimentController(experiment=exp, experiment_config=ecfg)

        sess = exp.session_set.first()
        # the session will appear as unfinished because in the dataset
        # imported, there are missing entries (not all subjects voted on all
        # stimuli), and it was assumed that a subject should have voted
        # for all stimuli in one session (which is not a practical assumption
        # in practice)
        self.assertEqual(ec.get_session_status(sess), SessionStatus.PARTIALLY_FINISHED)

        sinfo: dict = ec.get_session_info(sess)
        self.assertEqual(sinfo['subject'], 'VMAF_Plus_DB-andrewp')
        self.assertEqual(sinfo['session_id'], 1)
        self.assertEqual(sinfo['rounds'][-1]['round_id'], 31)
        self.assertEqual(sinfo['rounds'][-1]['stimulusgroup_id'], 52)
        self.assertEqual(sinfo['rounds'][-1]['stimulusvotegroups'][0]['stimulusvotegroup_id'], 52)
        self.assertEqual(sinfo['rounds'][-2]['round_id'], 30)
        self.assertEqual(sinfo['rounds'][-2]['stimulusgroup_id'], 51)
        self.assertEqual(sinfo['rounds'][-2]['stimulusvotegroups'][0]['stimulusvotegroup_id'], 51)
        self.assertEqual(sinfo['rounds'][-2]['stimulusvotegroups'][0]['vote'], 47.0)

        einfo: dict = ec.get_experiment_info()
        self.assertEqual(einfo['title'], 'VMAF_Plus_DB')
        self.assertEqual(einfo['description'], None)
        self.assertEqual(len(einfo['sessions']), 41)
        self.assertEqual(len(einfo['sessions'][0]['rounds']), 32)
        self.assertEqual(len(einfo['stimuli_info']['contents']), 3)
        self.assertEqual(len(einfo['stimuli_info']['stimuli']), 32)
        self.assertEqual(len(einfo['stimuli_info']['stimulusvotegroups']), 32)
        self.assertEqual(len(einfo['stimuli_info']['stimulusgroups']), 32)

        df = ExperimentController.denormalize_experiment_info(einfo)
        self.assertEqual(df.shape, (1312, 8))

        sess2 = ExperimentUtils.add_session_to_experiment(
            exp.title, 'zli', config=config, skip_path_check=True)
        self.assertEqual(sess2.round_set.count(), 32)
        self.assertEqual(sess2.subject.user.username, 'zli')

        subj = Subject()
        subj.save()
        ec.add_session(subj)
        sess2 = exp.session_set.get(subject=subj)
        self.assertEqual(ec.get_session_status(sess2), SessionStatus.INITIALIZED)

        ExperimentUtils.update_subject_for_session(
            exp.title, exp.session_set.all()[3].id, 'zli',
            config=config, skip_path_check=True)
        sess_3rd = exp.session_set.all()[3]
        self.assertEqual(sess_3rd.subject.user.username, 'zli')
        self.assertEqual(ec.get_session_status(sess_3rd), SessionStatus.PARTIALLY_FINISHED)

        ExperimentUtils.reset_unfinished_session(exp.title, sess_3rd.id,
                                                 config=config,
                                                 skip_path_check=True)
        self.assertEqual(ec.get_session_status(sess_3rd), SessionStatus.INITIALIZED)

        steps = ec.get_session_steps(sess)
        self.assertEqual(len(steps), 33)
        self.assertEqual(steps[1], {'context': {'stimulusgroup_id': 1000}, 'position': {'round_id': 0}})
        self.assertEqual(steps[-1], {'context': {'stimulusgroup_id': 52}, 'position': {'round_id': 31}})


class TestWriteExperimentConfigFile(TestCase):

    def setUp(self) -> None:
        self.config_filedir = NestConfig.tests_workdir_path(
            'test_write_experiment_config_file')

    def tearDown(self):
        shutil.rmtree(self.config_filedir)

    def test_populate_nest_nflx_public(self):
        dataset_path = NestConfig.tests_resource_path(
            'NFLX_dataset_public_raw_last4outliers_tiny.py')
        scale = 'FIVE_POINT'
        dataset = import_python_file(dataset_path)
        import_sureal_dataset(dataset, scale,
                              write_config=True,
                              config_filedir=self.config_filedir)

        # verify
        with open(os.path.join(self.config_filedir, "NFLX_public.json")) as fp:
            cfg = json.load(fp)
            self.assertEqual(len(cfg['stimulus_config']['stimulusgroups']), 28)


class TestCreateExperiment(TestCase):
    def setUp(self) -> None:
        self.config_filedir = os.path.dirname(NestSite.get_experiment_config_filepath("xxx", is_test=True))

    def tearDown(self):
        shutil.rmtree(self.config_filedir)

    def test_create_experiment(self):
        ec: ExperimentController = ExperimentUtils.create_experiment(
            experiment_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x.json'),
            experimenter_username='lukas@catflix.com',
            is_test=True,
            random_seed=1,
            experiment_title='io_tests.TestCreateExperiment.test_create_experiment')
        self.assertEqual(ec.experiment.title, 'io_tests.TestCreateExperiment.test_create_experiment')
        self.assertEqual(ec.experiment_config.random_seed, 1)
        self.assertEqual(ec.experiment.experimenters.first(),
                         Experimenter.find_by_username('lukas@catflix.com'))
        config_filepath = NestSite.get_experiment_config_filepath(ec.experiment_config.title, is_test=True)
        self.assertTrue(os.path.exists(config_filepath))
        with open(config_filepath, 'rt') as fp:
            config = json.load(fp)
        self.assertEqual(config['experiment_config']['random_seed'], 1)
        self.assertEqual(config['experiment_config']['title'],
                         'io_tests.TestCreateExperiment.test_create_experiment')

        subj = Subject.objects.create(name='Netflix_Noise_1')
        ec.add_session(subj)
        sess = ec.experiment.session_set.get(subject=subj)
        self.assertEqual(ec.get_session_status(sess), SessionStatus.INITIALIZED)

        subj2 = Subject.objects.create(name='Netflix_Noise_2')
        ec.add_session(subj2)
        sess2 = ec.experiment.session_set.get(subject=subj2)
        self.assertEqual(ec.get_session_status(sess2), SessionStatus.INITIALIZED)

        sinfo = ec.get_session_info(sess)
        self.assertEqual(
            sinfo,
            {'session_id': 1, 'subject': 'Netflix_Noise_1', 'rounds': [
                {'round_id': 0, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 1}]},
                {'round_id': 1, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0}]}
            ]})

        sinfo2 = ec.get_session_info(sess2)
        self.assertEqual(
            sinfo2,
            {'session_id': 2, 'subject': 'Netflix_Noise_2', 'rounds': [
                {'round_id': 0, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0}]},
                {'round_id': 1, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 1}]}
            ]})

        df = ESUtilities.export_playlist_ES_noise_study_style(ec)
        self.assertEqual(df.columns.to_list(),
                         ['participant_id', 'content', 'noise_level', 'interval', 'trial', 'Filename'])
        self.assertEqual(df.iloc[0].to_list(),
                         ['Netflix_Noise_1', 0, -1, 'A', 0, '/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4'])  # noqa E501
        self.assertEqual(df.iloc[1].to_list(),
                         ['Netflix_Noise_1', 0, -1, 'A', 1, '/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4'])  # noqa E501
        self.assertEqual(df.iloc[2].to_list(),
                         ['Netflix_Noise_2', 0, -1, 'A', 0, '/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4'])  # noqa E501
        self.assertEqual(df.iloc[3].to_list(),
                         ['Netflix_Noise_2', 0, -1, 'A', 1, '/media/mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4'])  # noqa E501

        self.assertEqual(Experiment.objects.count(), 1)
        self.assertEqual(Session.objects.count(), 2)
        self.assertEqual(Round.objects.count(), 4)
        self.assertEqual(Vote.objects.count(), 0)
        self.assertEqual(ExperimentRegister.objects.count(), 1)
        self.assertEqual(StimulusGroup.objects.count(), 2)
        self.assertEqual(StimulusVoteGroup.objects.count(), 2)
        self.assertEqual(VoteRegister.objects.count(), 2)
        self.assertEqual(Stimulus.objects.count(), 2)
        self.assertEqual(Content.objects.count(), 1)
        self.assertEqual(Condition.objects.count(), 0)

        ExperimentUtils.delete_experiment_by_title(
            experiment_title='io_tests.TestCreateExperiment.test_create_experiment',
            is_test=True)

        self.assertEqual(Experiment.objects.count(), 0)
        self.assertEqual(Session.objects.count(), 0)
        self.assertEqual(Round.objects.count(), 0)
        self.assertEqual(Vote.objects.count(), 0)
        self.assertEqual(ExperimentRegister.objects.count(), 0)
        self.assertEqual(StimulusGroup.objects.count(), 0)
        self.assertEqual(StimulusVoteGroup.objects.count(), 0)
        self.assertEqual(VoteRegister.objects.count(), 0)
        self.assertEqual(Stimulus.objects.count(), 0)
        self.assertEqual(Content.objects.count(), 0)
        self.assertEqual(Condition.objects.count(), 0)

    def test_create_experiment_es_noise_study(self):
        ec: ExperimentController = ExperimentUtils.create_experiment(
            experiment_config_filepath=NestConfig.tests_resource_path('es_noise_study_test1a1b_dcr_with_training_and_reliability.json'),
            experimenter_username='lukas@catflix.com',
            is_test=True,
            random_seed=1,
            experiment_title='io_tests.TestCreateExperiment.test_create_experiment_es_noise_study')

        subj = Subject.objects.create(name='Netflix_Noise_1')
        ec.add_session(subj)
        sess = ec.experiment.session_set.get(subject=subj)
        self.assertEqual(ec.get_session_status(sess), SessionStatus.INITIALIZED)

        subj2 = Subject.objects.create(name='Netflix_Noise_2')
        ec.add_session(subj2)
        sess2 = ec.experiment.session_set.get(subject=subj2)
        self.assertEqual(ec.get_session_status(sess2), SessionStatus.INITIALIZED)

        df = ESUtilities.export_playlist_ES_noise_study_style(ec)
        # df.to_csv('test.csv', index=False)
        self.assertEqual(df.columns.to_list(),
                         ['participant_id', 'content', 'noise_level', 'interval', 'trial', 'Filename'])
        self.assertEqual(df.iloc[0].to_list(),
                         ['Netflix_Noise_1', 1, -1, 'A', 0, '/media/mp4/es_noise_study/test1a1b/breaking-bad-season-1-gray-mat-filmgrain-subj-test-prep-101-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp18_vmaf93.08_psnr36.88_kbps115614.98_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[1].to_list(),
                         ['Netflix_Noise_1', 1, -1, 'B', 0, '/media/mp4/es_noise_study/test1a1b/breaking-bad-season-1-gray-mat-filmgrain-subj-test-prep-101-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp18_vmaf93.08_psnr36.88_kbps115614.98_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[2].to_list(),
                         ['Netflix_Noise_1', 1, 0, 'A', 1, '/media/mp4/es_noise_study/test1a1b/breaking-bad-season-1-gray-mat-filmgrain-subj-test-prep-101-beamr5-20211020aopalach-__beamr5__3840_2160__ni_denoise_filter_before_scale__qp18_vmaf83.17_psnr34.15_kbps28163.51_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[3].to_list(),
                         ['Netflix_Noise_1', 1, -1, 'B', 1, '/media/mp4/es_noise_study/test1a1b/breaking-bad-season-1-gray-mat-filmgrain-subj-test-prep-101-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp18_vmaf93.08_psnr36.88_kbps115614.98_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[10].to_list(),
                         ['Netflix_Noise_1', 0, 5, 'A', 5, '/media/mp4/es_noise_study/test1a1b/false-as-water-filmgrain-subj-test-prep-101b-beamr5-20211020aopalach-__beamr5__3840_2160__ni_denoise_with_target_noise_rmse_5_filter_before_scale__qp18_vmaf68.47_psnr33.67_kbps90682.35_fps24.0.mp4'])  # noqa E501
        self.assertEqual(df.iloc[-1].to_list(),
                         ['Netflix_Noise_2', 2, -1, 'B', 5, '/media/mp4/es_noise_study/test1a1b/PeeWeeS2Store-underVMAF-A-filmgrain-subj-test-prep-26-beamr5-20211007-__beamr5__960_540__none__qp37_vmaf26.91_psnr27.9_kbps283.91_fps29.97002997002997'])  # noqa E501

    def test_export_playlist_ES_noise_study_style_shuffle_intervals(self):
        ec: ExperimentController = ExperimentUtils.create_experiment(
            experiment_config_filepath=NestConfig.tests_resource_path('es_noise_study_test1a1b_dcr_with_training_and_reliability.json'),
            experimenter_username='lukas@catflix.com',
            is_test=True,
            random_seed=1,
            experiment_title='io_tests.TestCreateExperiment.test_export_playlist_ES_noise_study_style_shuffle_intervals')

        subj = Subject.objects.create(name='Netflix_Noise_1')
        ec.add_session(subj)
        sess = ec.experiment.session_set.get(subject=subj)
        self.assertEqual(ec.get_session_status(sess), SessionStatus.INITIALIZED)

        subj2 = Subject.objects.create(name='Netflix_Noise_2')
        ec.add_session(subj2)
        sess2 = ec.experiment.session_set.get(subject=subj2)
        self.assertEqual(ec.get_session_status(sess2), SessionStatus.INITIALIZED)

        random.seed(1)
        df = ESUtilities.export_playlist_ES_noise_study_style(ec, shuffle_intervals=True)
        self.assertEqual(df.columns.to_list(),
                         ['participant_id', 'content', 'noise_level', 'interval', 'trial', 'Filename'])
        self.assertEqual(df.iloc[0].to_list(),
                         ['Netflix_Noise_1', 1, -1, 'A', 0, '/media/mp4/es_noise_study/test1a1b/breaking-bad-season-1-gray-mat-filmgrain-subj-test-prep-101-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp18_vmaf93.08_psnr36.88_kbps115614.98_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[1].to_list(),
                         ['Netflix_Noise_1', 1, -1, 'B', 0, '/media/mp4/es_noise_study/test1a1b/breaking-bad-season-1-gray-mat-filmgrain-subj-test-prep-101-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp18_vmaf93.08_psnr36.88_kbps115614.98_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[2].to_list(),
                         ['Netflix_Noise_1', 1, -1, 'A', 1, '/media/mp4/es_noise_study/test1a1b/breaking-bad-season-1-gray-mat-filmgrain-subj-test-prep-101-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp18_vmaf93.08_psnr36.88_kbps115614.98_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[3].to_list(),
                         ['Netflix_Noise_1', 1, 0, 'B', 1, '/media/mp4/es_noise_study/test1a1b/breaking-bad-season-1-gray-mat-filmgrain-subj-test-prep-101-beamr5-20211020aopalach-__beamr5__3840_2160__ni_denoise_filter_before_scale__qp18_vmaf83.17_psnr34.15_kbps28163.51_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[10].to_list(),
                         ['Netflix_Noise_1', 0, 5, 'A', 5, '/media/mp4/es_noise_study/test1a1b/false-as-water-filmgrain-subj-test-prep-101b-beamr5-20211020aopalach-__beamr5__3840_2160__ni_denoise_with_target_noise_rmse_5_filter_before_scale__qp18_vmaf68.47_psnr33.67_kbps90682.35_fps24.0.mp4'])  # noqa E501
        self.assertEqual(df.iloc[-1].to_list(),
                         ['Netflix_Noise_2', 2, -1, 'B', 5, '/media/mp4/es_noise_study/test1a1b/PeeWeeS2Store-underVMAF-A-filmgrain-subj-test-prep-26-beamr5-20211007-__beamr5__960_540__none__qp37_vmaf26.91_psnr27.9_kbps283.91_fps29.97002997002997'])  # noqa E501

    def test_export_playlist_ES_noise_study_test2_samviq_style_shuffle_intervals(self):
        ec: ExperimentController = ExperimentUtils.create_experiment(
            experiment_config_filepath=NestConfig.tests_resource_path('es_noise_study_test2_small.json'),
            experimenter_username='lukas@catflix.com',
            is_test=True,
            random_seed=1,
            experiment_title='io_tests.TestCreateExperiment.test_export_playlist_ES_noise_study_test2_samviq_style_shuffle_intervals')

        subj = Subject.objects.create(name='Netflix_Noise_1')
        ec.add_session(subj)
        sess = ec.experiment.session_set.get(subject=subj)
        self.assertEqual(ec.get_session_status(sess), SessionStatus.INITIALIZED)

        subj2 = Subject.objects.create(name='Netflix_Noise_2')
        ec.add_session(subj2)
        sess2 = ec.experiment.session_set.get(subject=subj2)
        self.assertEqual(ec.get_session_status(sess2), SessionStatus.INITIALIZED)

        random.seed(1)
        df = ESUtilities.export_playlist_ES_noise_study_style(ec, shuffle_intervals=True, methodology='samviq')
        self.assertEqual(len(df.index), 40)

        self.assertEqual(df.columns.to_list(),
                         ['participant_id', 'content', 'noise_level', 'interval', 'trial', 'Filename'])
        self.assertEqual(df.iloc[0].to_list(),
                         ['Netflix_Noise_1', 0, -1, 'A', 0, '/Users/zli/Data/noise_study/projects_6sec_07262021/other/encode_selection/test2/training/taxi-driver-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp37_vmaf70.18_psnr32.6_kbps1203.85_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[1].to_list(),
                         ['Netflix_Noise_1', 0, -1, 'B', 0, '/Users/zli/Data/noise_study/projects_6sec_07262021/other/encode_selection/test2/training/taxi-driver-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp25_vmaf90.48_psnr35.28_kbps20330.72_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[2].to_list(),
                         ['Netflix_Noise_1', 0, -1, 'C', 0, '/Users/zli/Data/noise_study/projects_6sec_07262021/other/encode_selection/test2/training/taxi-driver-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp31_vmaf81.11_psnr33.89_kbps2924.92_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[3].to_list(),
                         ['Netflix_Noise_1', 0, -1, 'D', 0, '/Users/zli/Data/noise_study/projects_6sec_07262021/other/encode_selection/test2/training/taxi-driver-filmgrain-subj-test-prep-102-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp28_vmaf85.76_psnr34.37_kbps6643.19_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[4].to_list(),
                         ['Netflix_Noise_1', 0, -1, 'E', 0, '/Users/zli/Data/noise_study/projects_6sec_07262021/other/encode_selection/test2/training/reference/taxi-driver-filmgrain-subj-test-prep-102-4-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp20_vmaf96.07_psnr37.97_kbps78842.2_fps23.976023976023978.mp4'])  # noqa E501
        self.assertEqual(df.iloc[-1].to_list(),
                         ['Netflix_Noise_2', 3, -1, 'E', 3, '/Users/zli/Data/noise_study/projects_6sec_07262021/other/encode_selection/test2/testing/reference/groundhog-day-filmgrain-subj-test-prep-102-3-beamr5-20211020aopalach-__beamr5__3840_2160__none__qp18_vmaf93.49_psnr37.44_kbps86656.13_fps23.976023976023978.mp4'])  # noqa E501


class TestValidateConfig(TestCase):

    def test_validate_config(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_x.json'))

    def test_validate_config_dcr(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr.json'))

    def test_validate_config_mlds(self):
        with self.assertRaises(AssertionError):
            ExperimentUtils.validate_config(
                NestConfig.tests_resource_path('cvxhull_subjexp_toy_mlds.json'))

    def test_validate_config_mlds_bad(self):
        with self.assertRaises(AssertionError):
            ExperimentUtils.validate_config(
                NestConfig.tests_resource_path('cvxhull_subjexp_toy_mlds_bad.json'))

    def test_validate_config_dcr11d(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr11d.json'))

    def test_validate_config_dcr3d_standard(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr3d_standard.json'))

    def test_validate_config_dcr11d_standard(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr11d_standard.json'))

    def test_validate_config_dcr11d_standard_bad(self):
        with self.assertRaises(AssertionError):
            ExperimentUtils.validate_config(
                NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr11d_standard_bad.json'))

    def test_validate_config_dcr11d_bad_choices(self):
        with self.assertRaises(AssertionError):
            ExperimentUtils.validate_config(
                NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr11d_bad_choices.json'))

    def test_validate_config_samviq(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_samviq.json'))

    def test_validate_config_samviq5d(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_samviq5d.json'))

    def test_validate_config_acr5c(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_acr5c.json'))

    def test_validate_config_acr5c_standard(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_acr5c_standard.json'))

    def test_validate_config_tafc(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_tafc.json'))

    def test_validate_config_tafc_standard(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_tafc_standard.json'))

    def test_validate_config_ccr(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_ccr.json'))

    def test_validate_config_ccr_standard(self):
        ExperimentUtils.validate_config(
            NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_ccr_standard.json'))


class TestUserUtils(TestCase):

    def test_create_user_and_set_password(self):
        UserUtils.create_user('zli')
        user = User.objects.get(username='zli')
        self.assertFalse(user.is_staff)
        self.assertFalse(user.is_superuser)

        UserUtils.create_user('lkrasula', is_staff=True)
        user2 = User.objects.get(username='lkrasula')
        self.assertTrue(user2.is_staff)
        self.assertFalse(user2.is_superuser)

        UserUtils.create_user('christosb', is_superuser=True)
        user3 = User.objects.get(username='christosb')
        self.assertFalse(user3.is_staff)
        self.assertTrue(user3.is_superuser)

        UserUtils.set_password('zli', 'abc123')
        UserUtils.set_password('lkrasula', 'abc123')
        UserUtils.set_password('christosb', 'abc123')
        self.assertTrue(User.objects.get(username='zli').check_password('abc123'))
        self.assertTrue(User.objects.get(username='lkrasula').check_password('abc123'))
        self.assertTrue(User.objects.get(username='christosb').check_password('abc123'))


class TestConfigFileGeneration(TestCase):

    def setUp(self):
        self.output_config_json_filepath = \
            NestConfig.tests_workdir_path('TestConfigFileGeneration.json')

    def tearDown(self):
        if os.path.exists(self.output_config_json_filepath):
            os.remove(self.output_config_json_filepath)

    def populate_stimuli(self, filedir, mode, d_contents, d_stimuli,
                         d_stimulusgroups, d_stimulusvotegroups, prioritized,
                         blocklist_stimulusgroup_ids, training_round_ids):
        for path in glob.glob(os.path.join(filedir, '*.mp4')):
            content = os.path.basename(path).split('__')[0]
            content_id = ExperimentConfigFileUtils. \
                add_to_content_dict(content, d_contents)
            stimulus_id = ExperimentConfigFileUtils. \
                add_to_stimulus_dict(map_media_local_to_url(path), content_id, d_stimuli)
            stimulus_ids = [stimulus_id]  # single stimulus
            stimulusvotegroup_id = ExperimentConfigFileUtils. \
                add_to_stimulusvotegroup_dict(stimulus_ids,
                                              d_stimulusvotegroups)
            stimulusvotegroup_ids = [stimulusvotegroup_id]  # one vote per round
            stimulusgroup_id = ExperimentConfigFileUtils. \
                add_to_stimulusgroup_dict(stimulusvotegroup_ids,
                                          d_stimulusgroups)
            ExperimentConfigFileUtils.populate_prioritized_etc(
                stimulusgroup_id, mode, prioritized,
                blocklist_stimulusgroup_ids,
                training_round_ids)

    def test_generate_config_file_acr(self):

        d_contents = dict()
        d_stimuli = dict()
        d_stimulusvotegroups = dict()
        d_stimulusgroups = dict()

        prioritized = list()
        blocklist_stimulusgroup_ids = list()
        training_round_ids = list()

        training_filedir = NestConfig.media_path('mp4', 'cvxhull_subjexp', 'You')
        reliability_filedir = NestConfig.media_path('mp4', 'cvxhull_subjexp', 'GOViveATu')
        testing_filedir = NestConfig.media_path('mp4', 'cvxhull_subjexp', 'CT')

        self.populate_stimuli(training_filedir, 'training',
                              d_contents, d_stimuli, d_stimulusgroups, d_stimulusvotegroups,
                              prioritized, blocklist_stimulusgroup_ids, training_round_ids)
        self.populate_stimuli(reliability_filedir, 'reliability',
                              d_contents, d_stimuli, d_stimulusgroups, d_stimulusvotegroups,
                              prioritized, blocklist_stimulusgroup_ids, training_round_ids)
        self.populate_stimuli(testing_filedir, 'testing',
                              d_contents, d_stimuli, d_stimulusgroups, d_stimulusvotegroups,
                              prioritized, blocklist_stimulusgroup_ids, training_round_ids)

        additions = [
            {
                "position": {
                    "round_id": 0,
                    "before_or_after": "before"
                },
                "context": {
                    "title": "Instruction",
                    "text_html": "<p> This test involves evaluating the perceptual quality of videos. Please <b> set the browser in full-screen mode (shortcut: Control + Command + F) </b>. Please sit in a position such that your eyes are roughly aligned with the straight line marked on the floor. </p> <p> In each round, you are presented with two videos: Video A and Video B. Watch the videos and vote on which video has the better quality. </p> <p> There is no audio. You are expected to evaluate the <b> video quality </b> only. </p> <p> Videoss will loop over in the order of A -> B -> A ... up to <b> three times </b>. After the first time, you can <b> press \"space\" key or double click mouse </b> to skip to the voting page. </p>",  # noqa E501
                    "actions_html": "\n            <p>\n                <a class=\"button\" href=\"{action_url}\"  id=\"start\">Start Test</a>\n            </p>\n            "  # noqa E501
                }
            },
            {
                "position": {
                    "round_id": 0,
                    "before_or_after": "before"
                },
                "context": {
                    "title": "Training",
                    "text_html": " <p> To get you familiar with the test procedure, let's start with some training before kicking off the test officially. </p> ",  # noqa E501
                    "actions_html": "\n            <p>\n                <a class=\"button\" href=\"{action_url}\"  id=\"start\">Start Training</a>\n            </p>\n            "  # noqa E501
                }
            },
            {
                "position": {
                    "round_id": len(list(glob.glob(os.path.join(training_filedir, '*.mp4')))),
                    "before_or_after": "before"
                },
                "context": {
                    "title": "Start the test",
                    "text_html": " <p> Now let's officially start the test. </p> ",
                    "actions_html": "\n            <p>\n                <a class=\"button\" href=\"{action_url}\"  id=\"start\">Start Test</a>\n            </p>\n            "  # noqa E501
                }
            }
        ]

        round_context = {
            'video_display_percentage': 75,
            'video_show_controls': True,
        }

        stimulus_config = dict()
        stimulus_config['contents'] = list(d_contents.values())
        stimulus_config['stimuli'] = list(d_stimuli.values())
        stimulus_config['stimulusvotegroups'] = list(d_stimulusvotegroups.values())
        stimulus_config['stimulusgroups'] = list(d_stimulusgroups.values())

        experiment_config = dict()
        experiment_config['title'] = 'cvxhull_subjexp'
        experiment_config['description'] = 'convex hull subject test'
        experiment_config['vote_scale'] = "FIVE_POINT"
        experiment_config['methodology'] = 'acr'
        experiment_config['rounds_per_session'] = 16  # 8 stimuli, each twice
        experiment_config['random_seed'] = None
        experiment_config['prioritized'] = prioritized
        experiment_config['blocklist_stimulusgroup_ids'] = blocklist_stimulusgroup_ids
        experiment_config['training_round_ids'] = training_round_ids
        experiment_config['additions'] = additions
        experiment_config['round_context'] = round_context

        config = dict()
        config['stimulus_config'] = stimulus_config
        config['experiment_config'] = experiment_config

        config_json = json.dumps(config, indent=2)
        with open(self.output_config_json_filepath, 'wt') as f:
            f.write(config_json)

        ExperimentUtils.validate_config(self.output_config_json_filepath)

nest_site/nest/tests/io_tests.py
Netflix/e2nest
import datetime

from django.contrib.auth.models import User
from django.db.utils import IntegrityError
from django.test import TestCase
from django.utils import timezone
from nest.models import CcrThreePointVote, Condition, Content, ElevenPointVote, Experiment, Experimenter, \
    ExperimentRegister, FivePointVote, Round, Session, SevenPointVote, Stimulus, StimulusGroup, StimulusVoteGroup, \
    Subject, TafcVote, VoteRegister, Zero2HundredVote


class TestModels(TestCase):

    def test_saved(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        self.assertFalse(e.saved)
        e.save()
        self.assertTrue(e.saved)
        e2 = Experiment(title='Zhi ACR 2')
        e2.save()
        self.assertTrue(e2.saved)

    def test_deactivate(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        self.assertTrue(e.is_active)
        e.deactivate()
        self.assertFalse(e.is_active)

    def test_deactivate_past(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment",
                       deactivate_date=(timezone.now() -
                                        datetime.timedelta(days=1)))
        self.assertFalse(e.is_active)

    def test_deactivate_future(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment",
                       deactivate_date=(timezone.now() +
                                        datetime.timedelta(days=1)))
        self.assertTrue(e.is_active)

    def test_experiment(self):
        self.assertEqual(len(Experiment.objects.all()), 0)
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        self.assertEqual(len(Experiment.objects.all()), 0)
        e.save()
        self.assertEqual(len(Experiment.objects.all()), 1)
        self.assertEqual(e.title, 'Zhi ACR')
        self.assertEqual(e.description, "Zhi's ACR experiment")
        self.assertEqual(str(e), "Experiment 1 (Zhi ACR)")

    def test_experimenter(self):
        er = Experimenter(name='Zhi')
        er2 = Experimenter(name='Lukas')
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        er.save()
        er2.save()
        e.save()
        ExperimentRegister.objects.create(experiment=e, experimenter=er)
        ExperimentRegister.objects.create(experiment=e, experimenter=er2)
        self.assertEqual(e.experimenters.filter(id=er.id).count(), 1)
        self.assertEqual(e.experimenters.filter(id=er2.id).count(), 1)
        self.assertEqual(er.experiment_set.filter(id=e.id).count(), 1)
        self.assertEqual(er2.experiment_set.filter(id=e.id).count(), 1)
        er3 = Experimenter.find_by_username('Zhi')
        self.assertTrue(er3 is None)

        u = User.objects.create_user('lukas@catflix.com', password='abc123',
                                     is_staff=True)
        er3 = Experimenter(user=u)
        er3.save()
        er4 = Experimenter.find_by_username('lukas@catflix.com')
        self.assertTrue(er4 is not None)

    def test_experimenter_not_staff(self):
        user = User.objects.create_user('user', password='pass', is_staff=False)
        with self.assertRaises(AssertionError):
            Experimenter(name='Zhi', user=user)

    def test_experimenter_create_by_username(self):
        self.assertEqual(len(Experimenter.objects.all()), 0)
        self.assertEqual(len(Subject.objects.all()), 0)
        Experimenter.create_by_username('zli')
        self.assertEqual(len(Subject.objects.all()), 0)
        self.assertEqual(len(Experimenter.objects.all()), 1)
        self.assertEqual(len(User.objects.filter(username='zli', is_staff=True)), 1)
        self.assertEqual(len(User.objects.filter(username='zli', is_staff=False)), 0)
        with self.assertRaises(AssertionError):
            Experimenter.create_by_username('zli')
        Subject.create_by_username('zli')
        self.assertEqual(len(Subject.objects.all()), 1)

    def test_subject(self):
        self.assertEqual(len(Subject.objects.all()), 0)
        si = User.objects.create_user('lukas@catflix.com', password='abc123')
        s = Subject(name='Lukas', user=si)
        self.assertEqual(str(s), 'Subject None (Lukas, lukas@catflix.com)')
        si.save()
        s.save()
        self.assertEqual(str(s), 'Subject 1 (Lukas, lukas@catflix.com)')
        si2 = User.objects.create_user('zhi@catflix.com', password='xyz456')
        s2 = Subject(name='Zhi', user=si2)
        si2.save()
        s2.save()
        self.assertEqual(len(Subject.objects.all()), 2)
        si3 = User(username='zhi@catflix.com', password='xyz789')
        with self.assertRaises(IntegrityError):
            si3.save()

    def test_subject_without_name(self):
        si4 = User.objects.create_user('lukas4@catflix.com', password='abc1234')
        s4 = Subject(user=si4)
        self.assertEqual(str(s4), 'Subject None (lukas4@catflix.com)')

    def test_subject_find_by_username(self):
        self.assertEqual(len(Subject.objects.all()), 0)
        u = User.objects.create_user('lukas@catflix.com', password='abc123')
        s = Subject(name='Lukas', user=u)
        s.save()
        s2 = Subject.find_by_username('lukas@catflix.com')
        self.assertEqual(str(s2), 'Subject 1 (Lukas, lukas@catflix.com)')
        s3 = Subject.find_by_username('zli@catflix.com')
        self.assertTrue(s3 is None)
        u2 = User.objects.create_user('zli@catflix.com', password='abc123')
        s4 = Subject.find_by_username('zli@catflix.com')
        self.assertTrue(s4 is None)
        s5 = Subject(name='zli', user=u2)
        s5.save()
        s6 = Subject.find_by_username('zli@catflix.com')
        self.assertTrue(s6 is not None)
        self.assertEqual(str(s6), 'Subject 2 (zli, zli@catflix.com)')

    def test_subject_create_by_username(self):
        self.assertEqual(len(Subject.objects.all()), 0)
        self.assertEqual(len(Experimenter.objects.all()), 0)
        Subject.create_by_username('zli')
        self.assertEqual(len(Subject.objects.all()), 1)
        self.assertEqual(len(User.objects.filter(username='zli')), 1)
        with self.assertRaises(AssertionError):
            Subject.create_by_username('zli')
        Experimenter.create_by_username('zli')
        self.assertEqual(len(Experimenter.objects.all()), 1)

    def test_session(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        si = User.objects.create_user('lukas@catflix.com', password='abc123')
        s = Subject(name='Lukas', user=si)
        si2 = User.objects.create_user('zhi@catflix.com', password='xyz456')
        s2 = Subject(name='Zhi', user=si2)
        sess = Session(experiment=e, subject=s)
        sess2 = Session(experiment=e, subject=s2)
        self.assertEqual(str(sess),
                         "Session None (Experiment None (Zhi ACR), "
                         "Subject None (Lukas, lukas@catflix.com))")
        e.save()
        si.save()
        s.save()
        si2.save()
        s2.save()
        sess.save()
        sess2.save()
        self.assertEqual(str(sess),
                         "Session 1 (Experiment 1 (Zhi ACR), Subject 1 (Lukas, "
                         "lukas@catflix.com))")
        self.assertEqual(str(sess2),
                         "Session 2 (Experiment 1 (Zhi ACR), Subject 2 (Zhi, "
                         "zhi@catflix.com))")
        self.assertEqual(e.session_set.count(), 2)
        self.assertEqual(s.session_set.count(), 1)
        self.assertEqual(s2.session_set.count(), 1)

    def test_session_delete(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        si = User.objects.create_user('lukas@catflix.com', password='abc123')
        s = Subject(name='Lukas', user=si)
        sess = Session(experiment=e, subject=s)
        e.save()
        si.save()
        s.save()
        sess.save()
        Session.objects.get(id=1)
        Session.objects.get(id=1).delete()
        with self.assertRaises(Session.DoesNotExist):
            Session.objects.get(id=1)
        with self.assertRaises(Round.DoesNotExist):
            Round.objects.get(id=1)  # delete cascade: session -> round

    def test_experiment_delete(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        si = User.objects.create_user('lukas@catflix.com', password='abc123')
        s = Subject(name='Lukas', user=si)
        sess = Session(experiment=e, subject=s)
        e.save()
        si.save()
        s.save()
        sess.save()
        Experiment.objects.get(id=1)
        Experiment.objects.get(id=1).delete()
        with self.assertRaises(Experiment.DoesNotExist):
            Experiment.objects.get(id=1)
        with self.assertRaises(Session.DoesNotExist):
            Session.objects.get(id=1)  # delete cascade: experiment -> session

    def test_subject_delete(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        si = User.objects.create_user('lukas@catflix.com', password='abc123')
        s = Subject(name='Lukas', user=si)
        sess = Session(experiment=e, subject=s)
        e.save()
        si.save()
        s.save()
        sess.save()
        Subject.objects.get(id=s.id)
        Subject.objects.get(id=s.id).delete()
        with self.assertRaises(Subject.DoesNotExist):
            Subject.objects.get(id=s.id)
        sess2 = Session.objects.get(id=sess.id)
        # delete set null: subject -> session:
        self.assertEqual(sess2.subject, None)

    def test_round(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        si = User.objects.create_user('lukas@catflix.com', password='abc123')
        s = Subject(name='Lukas', user=si)
        sess = Session(experiment=e, subject=s)
        r = Round(session=sess, response_sec=10)
        e.save()
        si.save()
        s.save()
        sess.save()
        r.save()
        self.assertEqual(str(r),
                         "Round 1 (Session 1 (Experiment 1 (Zhi ACR), "
                         "Subject 1 (Lukas, lukas@catflix.com)), None, None)")
        sg = StimulusGroup(stimulusgroup_id=3)
        sg.save()
        r2 = Round(session=sess, round_id=4, stimulusgroup=sg, response_sec=4)
        r2.save()
        self.assertEqual(str(r2),
                         "Round 2 (Session 1 (Experiment 1 (Zhi ACR), "
                         "Subject 1 (Lukas, lukas@catflix.com)), 4, 3)")
        self.assertEqual(r.response_sec, 10)
        self.assertEqual(r2.response_sec, 4)

    def test_content_dataset(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        e.save()
        c = Content(name='convex hull dataset sparks01', content_id=3, experiment=e)
        c.save()
        self.assertEqual(c.content_id, 3)
        self.assertEqual(c.experiment, e)
        Content.objects.get(id=c.id).delete()
        with self.assertRaises(Content.DoesNotExist):
            Content.objects.get(id=c.id)

    def test_stimulus(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        e.save()
        cont = Content(name='sparks01')
        cond = Condition(name='AVC-Hi 950Kbps', condition_id=1, experiment=e)
        self.assertEqual(cond.condition_id, 1)
        self.assertEqual(cond.experiment, e)
        stim = Stimulus(content=cont, condition=cond, stimulus_id=4, experiment=e)
        cont.save()
        cond.save()
        stim.save()
        self.assertEqual(stim.content, cont)
        self.assertEqual(stim.condition, cond)
        self.assertEqual(stim.experiment, e)
        self.assertEqual(stim.stimulus_id, 4)
        self.assertEqual(str(stim), "Stimulus 1 (Content 1 (sparks01, None), "
                                    "Condition 1 (AVC-Hi 950Kbps), 4)")
        Content.objects.get(id=cont.id).delete()
        Condition.objects.get(id=cond.id).delete()
        stim2 = Stimulus.objects.get(id=stim.id)
        # delete set null: content -> stimulus:
        self.assertEqual(stim2.content, None)
        # delete set null: condition -> stimulus:
        self.assertEqual(stim2.condition, None)

    def test_5point_vote(self):
        v = FivePointVote(score=5)
        v.save()
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        si = User.objects.create_user('lukas@catflix.com', password='abc123')
        s = Subject(name='Lukas', user=si)
        sess = Session(experiment=e, subject=s)
        r = Round(session=sess, response_sec=5)
        v2 = FivePointVote(score=3, round=r)
        e.save()
        si.save()
        s.save()
        sess.save()
        r.save()
        v2.save()
        Experiment.objects.get(id=e.id).delete()
        with self.assertRaises(FivePointVote.DoesNotExist):
            # delete cascade: experiment -> session -> round -> vote:
            FivePointVote.objects.get(id=v2.id)

    def test_11point_vote(self):
        v = ElevenPointVote(score=11)
        v.save()
        with self.assertRaises(AssertionError):
            ElevenPointVote(score=0)

    def test_7point_vote(self):
        v = SevenPointVote(score=7)
        v.save()
        with self.assertRaises(AssertionError):
            SevenPointVote(score=0)

    def test_0to100_vote(self):
        v = Zero2HundredVote(score=50.1)
        v.save()
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        si = User.objects.create_user('lukas@catflix.com', password='abc123')
        s = Subject(name='Lukas', user=si)
        sess = Session(experiment=e, subject=s)
        r = Round(session=sess)
        v2 = Zero2HundredVote(score=100.0, round=r)
        e.save()
        si.save()
        s.save()
        sess.save()
        r.save()
        v2.save()
        Experiment.objects.get(id=e.id).delete()
        with self.assertRaises(Zero2HundredVote.DoesNotExist):
            # delete cascade: experiment -> session -> round -> vote:
            Zero2HundredVote.objects.get(id=v2.id)

    def test_5p_vote_validation(self):
        FivePointVote(score=2)
        with self.assertRaises(AssertionError):
            FivePointVote(score=0)
        with self.assertRaises(AssertionError):
            FivePointVote(score=1.5)

    def test_2afc_vote_validation(self):
        TafcVote(score=0)
        with self.assertRaises(AssertionError):
            TafcVote(score=2)
        with self.assertRaises(AssertionError):
            TafcVote(score=1.5)

    def test_ccr3p_vote_validation(self):
        CcrThreePointVote(score=0)
        CcrThreePointVote(score=1)
        CcrThreePointVote(score=2)
        with self.assertRaises(AssertionError):
            CcrThreePointVote(score=0.5)
        with self.assertRaises(AssertionError):
            CcrThreePointVote(score=3)

    def test_0to100_vote_validation(self):
        Zero2HundredVote(score=45.9)
        with self.assertRaises(AssertionError):
            TafcVote(score=-1)
        with self.assertRaises(AssertionError):
            TafcVote(score=101.5)

    def test_sg_svg(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        e.save()

        stim1 = Stimulus()
        stim2 = Stimulus()
        stim1.save()
        stim2.save()
        sg = StimulusGroup(stimulusgroup_id=1, experiment=e)
        sg.save()
        svg = StimulusVoteGroup.create_stimulusvotegroup_from_stimuli_pair(
            stim1, stim2,
            stimulusgroup=sg, stimulusvotegroup_id=1, experiment=e)
        self.assertEqual(str(sg), 'StimulusGroup 1 (1 : 1,2)')
        self.assertEqual(str(svg), 'StimulusVoteGroup 1 (1,2)')
        self.assertEqual(e.stimulusgroup_set.first(), sg)

        stim3 = Stimulus()
        stim3.save()
        svg2 = StimulusVoteGroup.create_stimulusvotegroup_from_stimuli_pair(
            stim3, stim2,
            stimulusgroup=sg)
        self.assertEqual(str(sg), 'StimulusGroup 1 (1 : 1,2,3)')
        self.assertEqual(str(svg2), 'StimulusVoteGroup 2 (2,3)')

    def test_svg_without_sg(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        e.save()
        stim1 = Stimulus()
        stim2 = Stimulus()
        stim1.save()
        stim2.save()
        svg = StimulusVoteGroup.create_stimulusvotegroup_from_stimuli_pair(
            stim1, stim2, stimulusvotegroup_id=1, experiment=e)

        self.assertEqual(str(svg), 'StimulusVoteGroup 1 (1,2)')
        self.assertTrue(svg.stimulusgroup is None)

        svg2 = StimulusVoteGroup.create_stimulusvotegroup_from_stimulus(
            stim2, stimulusvotegroup_id=1, experiment=e)

        self.assertEqual(str(svg2), 'StimulusVoteGroup 2 (2)')
        self.assertTrue(svg2.stimulusgroup is None)

    def test_5p_vote_stimuli(self):
        e = Experiment(title='Zhi ACR', description="Zhi's ACR experiment")
        si = User.objects.create_user('lukas@catflix.com', password='abc123')
        s = Subject(name='Lukas', user=si)
        sess = Session(experiment=e, subject=s)
        r = Round(session=sess)
        e.save()
        si.save()
        s.save()
        sess.save()
        r.save()

        stim1 = Stimulus()
        stim2 = Stimulus()
        stim1.save()
        stim2.save()

        # create svg with order (stim1, stim2)
        sg = StimulusGroup()
        sg.save()
        svg = StimulusVoteGroup.create_stimulusvotegroup_from_stimuli_pair(
            stim1, stim2, sg)

        # create svg2 with order (stim2, stim1)
        sg2 = StimulusGroup()
        sg2.save()
        svg2 = StimulusVoteGroup.create_stimulusvotegroup_from_stimuli_pair(
            stim2, stim1, sg2)

        # create svg3 with order (stim1, stim3)
        stim3 = Stimulus()
        stim3.save()
        sg3 = StimulusGroup()
        sg3.save()
        svg3 = StimulusVoteGroup.create_stimulusvotegroup_from_stimuli_pair(
            stim1, stim3, sg3)

        self.assertEqual(svg.stimulus.filter(id=stim1.id).count(), 1)
        self.assertEqual(svg.stimulus.filter(id=stim2.id).count(), 1)
        self.assertEqual(stim1.stimulusvotegroup_set.filter(id=svg.id).count(), 1)
        self.assertEqual(stim2.stimulusvotegroup_set.filter(id=svg.id).count(), 1)
        self.assertEqual(VoteRegister.objects
                         .filter(stimulusvotegroup=svg, stimulus=stim1)[0].stimulus_order, 1)
        self.assertEqual(VoteRegister.objects
                         .filter(stimulusvotegroup=svg, stimulus=stim2)[0].stimulus_order, 2)
        self.assertEqual(VoteRegister.objects
                         .filter(stimulusvotegroup=svg, stimulus_order=1)[0].id, 1)
        self.assertEqual(VoteRegister.objects
                         .filter(stimulusvotegroup=svg, stimulus_order=2)[0].id, 2)

        svgs_ = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim1, stim2)
        self.assertEqual(len(svgs_), 1)
        self.assertEqual(svg, svgs_[0])

        svgs2_ = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim2, stim1)
        self.assertEqual(len(svgs2_), 1)
        self.assertEqual(svg2, svgs2_[0])

        svgs3_ = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim1, stim3)
        self.assertEqual(len(svgs3_), 1)
        self.assertEqual(svg3, svgs3_[0])

        # create sg4 and svg4 with order (stim1, stim2). This is to simulate the case
        # that stim, stim2 got reused by another experiment:
        sg4 = StimulusGroup()
        sg4.save()
        svg4 = StimulusVoteGroup.create_stimulusvotegroup_from_stimuli_pair(
            stim1, stim2,
            stimulusgroup=sg4, stimulusvotegroup_id=0, experiment=e)
        svgs4_ = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim1, stim2)
        self.assertEqual(len(svgs4_), 2)
        self.assertTrue(svg in svgs4_)
        self.assertTrue(svg4 in svgs4_)
        self.assertEqual(svg4.stimulusvotegroup_id, 0)
        self.assertEqual(svg4.experiment, e)

        # create another single-stimulus svg
        sg5 = StimulusGroup()
        sg5.save()
        svg5 = StimulusVoteGroup.create_stimulusvotegroup_from_stimulus(
            stim1,
            stimulusgroup=sg5, stimulusvotegroup_id=1, experiment=e)
        svgs5_ = StimulusVoteGroup.find_stimulusvotegroups_from_stimulus(stim1)
        self.assertEqual(len(svgs5_), 4)
        self.assertTrue(svg5 in svgs5_)
        self.assertTrue(svg in svgs5_)
        self.assertTrue(svg4 in svgs5_)
        self.assertTrue(svg3 in svgs5_)
        self.assertFalse(svg2 in svgs5_)
        self.assertEqual(svg5.stimulusvotegroup_id, 1)
        self.assertEqual(svg5.experiment, e)

        # re-check svgs4_
        svgs4_2 = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim1, stim2)
        self.assertEqual(len(svgs4_2), 2)
        self.assertTrue(svg in svgs4_2)
        self.assertTrue(svg4 in svgs4_2)

nest_site/nest/tests/model_tests.py
Netflix/e2nest
import os
import unittest
from time import sleep

from django.contrib.auth.models import User
from django.contrib.staticfiles.testing import StaticLiveServerTestCase
from django.urls import reverse
from nest.config import NestConfig
from nest.io import ExperimentUtils
from nest.models import CcrThreePointVote, ElevenPointVote, FivePointVote, Round, SevenPointVote, Subject, TafcVote, \
    ThreePointVote, Vote, Zero2HundredVote
from nest.sites import NestSite
from nest_site import settings
from selenium import webdriver
from selenium.webdriver import Keys
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait

os.environ['PATH'] += os.pathsep + os.path.dirname(settings.CHROMEDRIVER_PATH)

MSG_INCORRECT_CHROMEDRIVER_PATH = \
    'CHROMEDRIVER_PATH must be correctly set to chromedriver in ' \
    'nest_site.settings, see https://chromedriver.chromium.org/home'


class LoginMixin(object):

    LOGIN_SUCCESS_MSG = 'Welcome'
    LOGOUT_SUCCESS_MSG = 'Thanks for spending some quality time with the Web site today.'

    def login(self):
        self.browser.get(self.live_server_url + reverse("nest:login"))
        self.browser.find_element(by='name', value='username').send_keys('user')
        self.browser.find_element(by='name', value='password').send_keys('pass')
        self.browser.find_element(by='xpath', value='//input[@value="Log in"]').click()
        WebDriverWait(self.browser, 10).until(
            lambda driver: driver.find_element('tag name', 'body'))
        self.assertTrue(self.LOGIN_SUCCESS_MSG in self.browser.page_source)

    def logout(self):
        self.browser.get(self.live_server_url + reverse("nest:logout"))
        WebDriverWait(self.browser, 10).until(
            lambda driver: driver.find_element('tag name', 'body'))
        self.assertTrue(self.LOGOUT_SUCCESS_MSG in self.browser.page_source)


@unittest.skipUnless(
    hasattr(settings, 'CHROMEDRIVER_PATH') and os.path.exists(settings.CHROMEDRIVER_PATH),
    MSG_INCORRECT_CHROMEDRIVER_PATH)
class TestBrowser(StaticLiveServerTestCase, LoginMixin):

    browser = None

    def setUp(self):
        super().setUp()
        self.user = User.objects.create_user('user', password='pass', is_staff=False)
        options = Options()
        options.add_argument("--disable-dev-shm-usage")
        options.add_argument("--no-sandbox")
        options.add_argument('--headless')
        self.browser = webdriver.Chrome(options=options)
        self.browser.implicitly_wait(10)

    def tearDown(self):
        self.browser.quit()
        self.user.delete()
        super().tearDown()

    def test_login_logout(self):
        self.login()
        self.logout()


@unittest.skipUnless(
    hasattr(settings, 'CHROMEDRIVER_PATH') and os.path.exists(settings.CHROMEDRIVER_PATH),
    MSG_INCORRECT_CHROMEDRIVER_PATH)
class TestBrowserWithWriteDataset(StaticLiveServerTestCase, LoginMixin):

    EXPERIMENT_TITLE = 'nest_browser_tests.TestBrowserWithWriteDataset'

    def setUp(self):
        super().setUp()
        self.config_filepath = NestSite.get_experiment_config_filepath(self.EXPERIMENT_TITLE, is_test=False)
        self.user = User.objects.create_user('user', password='pass', is_staff=False)
        options = Options()
        options.add_argument("--disable-dev-shm-usage")
        options.add_argument("--no-sandbox")
        options.add_argument('--headless')
        self.browser = webdriver.Chrome(options=options)
        self.browser.implicitly_wait(10)

    def tearDown(self):
        os.remove(self.config_filepath)
        self.user.delete()
        self.browser.quit()
        super().tearDown()

    def test_status_page(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_dcr.json'),
            is_test=False,
            random_seed=1,
            experiment_title=self.EXPERIMENT_TITLE)
        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)
        self.login()
        self.assertTrue(self.EXPERIMENT_TITLE in self.browser.page_source)
        self.assertTrue(str(sess.id) in self.browser.page_source)
        self.assertTrue('New' in self.browser.page_source)
        self.assertTrue('Start' in self.browser.page_source)

    def test_step_session_dcr(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_dcr.json'),
            is_test=False,
            random_seed=1,
            experiment_title=self.EXPERIMENT_TITLE)

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)
        self.login()
        self.browser.get(self.live_server_url + reverse('nest:start_session', kwargs={'session_id': sess.id}))
        self.browser.find_element(by='id', value='start').click()
        self.assertTrue('Round 1 of 2' in self.browser.page_source)
        self.browser.find_element(by='id', value='video_a').click()
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        self.browser.find_element(by='id', value='video_b').click()
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        self.browser.find_element(by='id', value='radio_dcr1').click()

        self.browser.find_element(by='id', value='submit').click()
        self.assertTrue('Round 2 of 2' in self.browser.page_source)
        self.browser.find_element(by='id', value='video_a').click()
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        self.browser.find_element(by='id', value='video_b').click()
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        self.browser.find_element(by='id', value='radio_dcr4').click()

        self.assertEqual(Vote.objects.count(), 0)

        self.browser.find_element(by='id', value='submit').click()
        self.assertTrue('Test done' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 4)
        self.assertEqual(FivePointVote.objects.first().score, 1)
        self.assertEqual(FivePointVote.objects.all()[1].score, 4)

        r1 = Round.objects.get(session=sess, round_id=0)
        r2 = Round.objects.get(session=sess, round_id=1)
        self.assertTrue(r1.response_sec > 0)
        self.assertTrue(r2.response_sec > 0)

    def test_step_session_dcr11d(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr11d.json'),
            is_test=False,
            random_seed=1,
            experiment_title=self.EXPERIMENT_TITLE)

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        self.login()
        sleep(0.2)
        self.browser.get(self.live_server_url + reverse('nest:start_session', kwargs={'session_id': sess.id}))
        sleep(0.2)
        self.browser.find_element(by='id', value='start').click()
        sleep(0.2)
        self.assertTrue('Round 1 of 2' in self.browser.page_source)
        sleep(0.2)
        self.browser.find_element(by='id', value='video_a').click()
        sleep(0.2)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.2)
        self.browser.find_element(by='id', value='video_b').click()
        sleep(0.2)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.2)
        self.browser.find_element(by='id', value='radio_dcr1').click()
        sleep(0.2)
        self.browser.find_element(by='id', value='submit').click()
        sleep(0.2)
        self.assertTrue('Round 2 of 2' in self.browser.page_source, f'page_source: {self.browser.page_source}')
        sleep(0.2)
        self.browser.find_element(by='id', value='video_a').click()
        sleep(0.2)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.2)
        self.browser.find_element(by='id', value='video_b').click()
        sleep(0.2)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.2)
        self.browser.find_element(by='id', value='radio_dcr4').click()
        sleep(0.2)

        self.assertEqual(Vote.objects.count(), 0)

        sleep(0.2)
        self.browser.find_element(by='id', value='submit').click()
        sleep(0.5)
        self.assertTrue('Test done' in self.browser.page_source, f'page_source: {self.browser.page_source}')

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(ElevenPointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 4)
        self.assertEqual(ElevenPointVote.objects.first().score, 1)
        self.assertEqual(ElevenPointVote.objects.all()[1].score, 4)

    def test_step_session_dcr7d_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr7d_standard.json'),
            is_test=False,
            random_seed=1,
            experiment_title=self.EXPERIMENT_TITLE)

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        self.login()
        self.browser.get(self.live_server_url + reverse('nest:start_session', kwargs={'session_id': sess.id}))
        self.browser.find_element(by='id', value='start').click()
        self.assertTrue('Round 1 of 2' in self.browser.page_source)

        sleep(0.5)  # 0.5 sec buffer
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)

        self.browser.find_element(by='id', value='radio_dcr1').click()
        self.browser.find_element(by='id', value='submit').click()
        self.assertTrue('Round 2 of 2' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 0)

        sleep(0.5)  # 0.5 sec buffer
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)

        self.browser.find_element(by='id', value='radio_dcr7').click()
        self.browser.find_element(by='id', value='submit').click()
        self.assertTrue('Test done' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(SevenPointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 7)
        self.assertEqual(SevenPointVote.objects.first().score, 1)
        self.assertEqual(SevenPointVote.objects.all()[1].score, 7)

    def test_step_session_dcr11d_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr11d_standard.json'),
            is_test=False,
            random_seed=1,
            experiment_title=self.EXPERIMENT_TITLE)

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        self.login()
        sleep(0.2)
        self.browser.get(self.live_server_url + reverse('nest:start_session', kwargs={'session_id': sess.id}))
        sleep(0.2)
        self.browser.find_element(by='id', value='start').click()
        sleep(0.2)
        self.assertTrue('Round 1 of 2' in self.browser.page_source)

        sleep(0.5)  # 0.5 sec buffer
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)

        sleep(0.2)
        self.browser.find_element(by='id', value='radio_dcr1').click()
        sleep(0.2)
        self.browser.find_element(by='id', value='submit').click()
        sleep(0.2)
        self.assertTrue('Round 2 of 2' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 0)

        sleep(0.5)  # 0.5 sec buffer
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.2)

        self.browser.find_element(by='id', value='radio_dcr4').click()
        sleep(0.2)
        self.browser.find_element(by='id', value='submit').click()
        sleep(0.2)
        self.assertTrue('Test done' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(ElevenPointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 4)
        self.assertEqual(ElevenPointVote.objects.first().score, 1)
        self.assertEqual(ElevenPointVote.objects.all()[1].score, 4)

    def test_step_session_dcr3d_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr3d_standard.json'),
            is_test=False,
            random_seed=1,
            experiment_title=self.EXPERIMENT_TITLE)

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        self.login()
        self.browser.get(self.live_server_url + reverse('nest:start_session', kwargs={'session_id': sess.id}))
        self.browser.find_element(by='id', value='start').click()
        self.assertTrue('Round 1 of 2' in self.browser.page_source)

        sleep(0.5)  # 0.5 sec buffer
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)

        self.browser.find_element(by='id', value='radio_dcr1').click()
        self.browser.find_element(by='id', value='submit').click()
        self.assertTrue('Round 2 of 2' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 0)

        sleep(0.5)  # 0.5 sec buffer
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)

        self.browser.find_element(by='id', value='radio_dcr3').click()
        self.browser.find_element(by='id', value='submit').click()
        self.assertTrue('Test done' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(ThreePointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 3)
        self.assertEqual(ThreePointVote.objects.first().score, 1)
        self.assertEqual(ThreePointVote.objects.all()[1].score, 3)

    def test_step_session_tafc_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_tafc_standard.json'),
            is_test=False,
            random_seed=1,
            experiment_title=self.EXPERIMENT_TITLE)

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        self.login()
        self.browser.get(self.live_server_url + reverse('nest:start_session', kwargs={'session_id': sess.id}))
        self.browser.find_element(by='id', value='start').click()
        self.assertTrue('Round 1 of 2' in self.browser.page_source)

        sleep(0.5)  # 0.5 sec buffer
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)

        button = self.browser.find_element(by='id', value='radio_ccr0')
        value = int(button.get_attribute("value"))
        button.click()
        self.browser.find_element(by='id', value='submit').click()
        self.assertTrue('Round 2 of 2' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 0)

        sleep(0.5)  # 0.5 sec buffer
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)

        button2 = self.browser.find_element(by='id', value='radio_ccr1')
        value2 = int(button2.get_attribute("value"))
        button2.click()
        self.browser.find_element(by='id', value='submit').click()
        self.assertTrue('Test done' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 0)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)
        self.assertEqual(TafcVote.objects.count(), 2)

        self.assertEqual(Vote.objects.first().score, value)
        self.assertEqual(Vote.objects.all()[1].score, value2)
        self.assertEqual(TafcVote.objects.first().score, value)
        self.assertEqual(TafcVote.objects.all()[1].score, value2)

    def test_step_session_samviq(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_samviq.json'),
            is_test=False,
            random_seed=1,
            experiment_title=self.EXPERIMENT_TITLE)

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        self.login()
        sleep(0.1)
        self.browser.get(self.live_server_url + reverse('nest:start_session', kwargs={'session_id': sess.id}))
        sleep(0.1)
        self.browser.find_element(by='id', value='start').click()
        sleep(0.1)
        self.assertTrue('Round 1 of 2' in self.browser.page_source)
        self.browser.find_element(by='id', value='video_ref').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)
        self.browser.find_element(by='id', value='video_0').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)
        self.browser.find_element(by='id', value='video_1').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)

        self.assertEqual(Vote.objects.count(), 0)

        self.browser.find_element(by='id', value='myRange_3').send_keys(Keys.RIGHT)
        sleep(0.1)
        self.browser.find_element(by='id', value='myRange_2').send_keys(Keys.RIGHT)
        sleep(0.1)

        self.browser.find_element(by='id', value='submit').click()
        sleep(0.1)

        self.assertEqual(Vote.objects.count(), 0)

        self.assertTrue('Round 2 of 2' in self.browser.page_source)
        self.browser.find_element(by='id', value='video_ref').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)
        self.browser.find_element(by='id', value='video_0').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)
        self.browser.find_element(by='id', value='video_1').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)

        self.assertEqual(Vote.objects.count(), 0)

        self.browser.find_element(by='id', value='submit').click()
        sleep(0.1)
        self.assertTrue('Test done' in self.browser.page_source, f'page_source: {self.browser.page_source}')

        self.assertEqual(Vote.objects.count(), 4)
        self.assertEqual(Vote.objects.all()[0].score, 51)
        self.assertEqual(Vote.objects.all()[1].score, 51)
        self.assertEqual(Vote.objects.all()[2].score, 50)
        self.assertEqual(Vote.objects.all()[3].score, 50)

        r1 = Round.objects.get(session=sess, round_id=0)
        r2 = Round.objects.get(session=sess, round_id=1)
        self.assertTrue(r1.response_sec > 0)
        self.assertTrue(r2.response_sec > 0)

    def test_step_session_samviq_with_preload(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_samviq_with_preload.json'),
            is_test=False,
            random_seed=1,
            experiment_title=self.EXPERIMENT_TITLE)

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        self.login()
        sleep(0.1)
        self.browser.get(self.live_server_url + reverse('nest:start_session', kwargs={'session_id': sess.id}))
        sleep(0.1)
        self.browser.find_element(by='id', value='start').click()
        self.assertTrue('Round 1 of 2' in self.browser.page_source)
        sleep(0.1)
        self.browser.find_element(by='id', value='video_ref').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)
        self.browser.find_element(by='id', value='video_0').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)
        self.browser.find_element(by='id', value='video_1').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)

        self.assertEqual(Vote.objects.count(), 0)

        self.browser.find_element(by='id', value='myRange_3').send_keys(Keys.RIGHT)
        sleep(0.1)
        self.browser.find_element(by='id', value='myRange_2').send_keys(Keys.RIGHT)
        sleep(0.1)

        self.browser.find_element(by='id', value='submit').click()
        sleep(0.1)

        self.assertEqual(Vote.objects.count(), 0)

        self.assertTrue('Round 2 of 2' in self.browser.page_source)
        sleep(0.1)
        self.browser.find_element(by='id', value='video_ref').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)
        self.browser.find_element(by='id', value='video_0').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)
        self.browser.find_element(by='id', value='video_1').click()
        sleep(0.1)
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)
        sleep(0.1)

        self.assertEqual(Vote.objects.count(), 0)

        self.browser.find_element(by='id', value='submit').click()
        sleep(0.1)
        self.assertTrue('Test done' in self.browser.page_source, f'page_source: {self.browser.page_source}')

        self.assertEqual(Vote.objects.count(), 4)
        self.assertEqual(Vote.objects.all()[0].score, 51)
        self.assertEqual(Vote.objects.all()[1].score, 51)
        self.assertEqual(Vote.objects.all()[2].score, 50)
        self.assertEqual(Vote.objects.all()[3].score, 50)

        r1 = Round.objects.get(session=sess, round_id=0)
        r2 = Round.objects.get(session=sess, round_id=1)
        self.assertTrue(r1.response_sec > 0)
        self.assertTrue(r2.response_sec > 0)

    def test_step_session_ccr_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_ccr_standard.json'),
            is_test=False,
            random_seed=1,
            experiment_title=self.EXPERIMENT_TITLE)

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        self.login()
        self.browser.get(self.live_server_url + reverse('nest:start_session', kwargs={'session_id': sess.id}))
        self.browser.find_element(by='id', value='start').click()
        self.assertTrue('Round 1 of 2' in self.browser.page_source)

        sleep(0.5)  # 0.5 sec buffer
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)

        button = self.browser.find_element(by='id', value='radio_ccr0')
        value = int(button.get_attribute("value"))
        button.click()
        self.browser.find_element(by='id', value='submit').click()
        self.assertTrue('Round 2 of 2' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 0)

        sleep(0.5)  # 0.5 sec buffer
        self.browser.find_element(by='tag name', value="body").send_keys(Keys.SPACE)

        button2 = self.browser.find_element(by='id', value='radio_ccr1')
        value2 = int(button2.get_attribute("value"))
        button2.click()
        self.browser.find_element(by='id', value='submit').click()
        self.assertTrue('Test done' in self.browser.page_source)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 0)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)
        self.assertEqual(CcrThreePointVote.objects.count(), 2)

        self.assertEqual(Vote.objects.first().score, value)
        self.assertEqual(Vote.objects.all()[1].score, value2)
        self.assertEqual(CcrThreePointVote.objects.first().score, value)
        self.assertEqual(CcrThreePointVote.objects.all()[1].score, value2)

nest_site/nest/tests/nest_browsertests.py
Netflix/e2nest
import logging
import os
import shutil

from django.contrib.auth.models import User
from django.test import TestCase
from django.urls import reverse
from nest.config import NestConfig
from nest.control import ExperimentController
from nest.io import ExperimentUtils, export_sureal_dataset
from nest.models import CcrThreePointVote, ElevenPointVote, FivePointVote, Round, SevenPointVote, Stimulus, \
    StimulusVoteGroup, Subject, TafcVote, ThreePointVote, Vote, Zero2HundredVote
from nest.sites import NestSite


class TestViews(TestCase):

    def setUp(self) -> None:
        self.user = User.objects.create_user('user', password='pass', is_staff=False)
        self.staff = User.objects.create_user('staff', password='pass', is_staff=True)

    def tearDown(self):
        self.user.delete()
        self.staff.delete()

    def test_login(self):
        response = self.client.get(reverse('nest:login'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:login'))
        self.assertEqual(response.status_code, 302)  # already logged in - redirect

    def test_logout(self):
        response = self.client.get(reverse('nest:logout'))
        self.assertEqual(response.status_code, 302)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:logout'))
        self.assertEqual(response.status_code, 200)

    def test_change_password(self):
        response = self.client.get(reverse('nest:password_change'))
        self.assertEqual(response.status_code, 302)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:password_change'))
        self.assertEqual(response.status_code, 200)

    def test_nestexp_with_login(self):
        response = self.client.get(reverse('admin:nestexp'))
        self.assertEqual(response.status_code, 302)

        login = self.client.login(username='user', password='paxx')
        self.assertFalse(login)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('admin:nestexp'))
        self.assertEqual(response.status_code, 302)

        login = self.client.login(username='staff', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('admin:nestexp'))
        self.assertEqual(response.status_code, 200)

    def test_index_with_login(self):
        response = self.client.get(reverse('nest:index'))
        self.assertEqual(response.status_code, 302)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:index'))
        self.assertEqual(response.status_code, 302)

    def test_status_with_login(self):
        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 302)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

    def test_instruction_with_login(self):
        response = self.client.get(reverse('nest:instruction_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='paxx')
        self.assertFalse(login)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:instruction_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='staff', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:instruction_demo'))
        self.assertEqual(response.status_code, 200)

    def test_acr_with_login(self):
        response = self.client.get(reverse('nest:acr_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:acr_demo'))
        self.assertEqual(response.status_code, 200)

    def test_acr5c_with_login(self):
        response = self.client.get(reverse('nest:acr5c_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:acr5c_demo'))
        self.assertEqual(response.status_code, 200)

    def test_acr5c_standard_with_login(self):
        response = self.client.get(reverse('nest:acr5c_standard_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:acr5c_standard_demo'))
        self.assertEqual(response.status_code, 200)

    def test_dcr_with_login(self):
        response = self.client.get(reverse('nest:dcr_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:dcr_demo'))
        self.assertEqual(response.status_code, 200)

    def test_tafc_with_login(self):
        response = self.client.get(reverse('nest:tafc_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:tafc_demo'))
        self.assertEqual(response.status_code, 200)

    def test_tafc_standard_with_login(self):
        response = self.client.get(reverse('nest:tafc_standard_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:tafc_standard_demo'))
        self.assertEqual(response.status_code, 200)

    def test_ccr_with_login(self):
        response = self.client.get(reverse('nest:ccr_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:ccr_demo'))
        self.assertEqual(response.status_code, 200)

    def test_ccr_standard_with_login(self):
        response = self.client.get(reverse('nest:ccr_standard_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:ccr_standard_demo'))
        self.assertEqual(response.status_code, 200)

    def test_dcr11d_with_login(self):
        response = self.client.get(reverse('nest:dcr11d_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:dcr11d_demo'))
        self.assertEqual(response.status_code, 200)

    def test_dcr11d_standard_with_login(self):
        response = self.client.get(reverse('nest:dcr11d_standard_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:dcr11d_standard_demo'))
        self.assertEqual(response.status_code, 200)

    def test_samviq_with_login(self):
        response = self.client.get(reverse('nest:samviq_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:samviq_demo'))
        self.assertEqual(response.status_code, 200)

    def test_samviq5d_with_login(self):
        response = self.client.get(reverse('nest:samviq5d_demo'))
        self.assertEqual(response.status_code, 200)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:samviq5d_demo'))
        self.assertEqual(response.status_code, 200)


class TestViewsWithWriteDataset(TestCase):

    def setUp(self) -> None:
        self.user = User.objects.create_user('user', password='pass', is_staff=False)
        self.staff = User.objects.create_user('staff', password='pass', is_staff=True)
        self.config_filedir = os.path.dirname(NestSite.get_experiment_config_filepath("xxx", is_test=True))
        logging.disable(logging.CRITICAL)

    def tearDown(self):
        self.user.delete()
        self.staff.delete()
        shutil.rmtree(self.config_filedir)
        logging.disable(logging.NOTSET)

    def test_start_and_step_session_with_login_and_cookie(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_start_and_step_session_with_login_and_cookie')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        # hit cookie_failed with 200, because haven't set_test_cookie()
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        # hit 302: redirect to step_session page
        self.assertEqual(response.status_code, 302)

        self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        s = self.client.session
        s.delete_test_cookie()
        s.save()
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        # hit cookie_failed with 200, because delete_test_cookie()
        self.assertEqual(response.status_code, 200)

    def test_step_session_with_login_and_cookie(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_with_login_and_cookie')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        # hit cookie_failed with 200, because haven't set_test_cookie()
        self.assertEqual(response.status_code, 200)

        self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        s = self.client.session
        s.delete_test_cookie()
        s.save()
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        # hit cookie_failed with 200, because delete_test_cookie()
        self.assertEqual(response.status_code, 200)

    def test_step_session(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session')

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(self.client.session['session_id'], 1)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'acr_1': '1'})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(self.client.session['session_id'], 1)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 1})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'acr_0': '4'})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(self.client.session['session_id'], 1)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 4})

        response = self.client.get(reverse('nest:cookie'))
        self.assertEqual(response.status_code, 200)
        self.assertTrue(""""score": {\n                    "0": 4""" in response.context_data['text_html'])

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 4)
        self.assertEqual(FivePointVote.objects.first().score, 1)
        self.assertEqual(FivePointVote.objects.all()[1].score, 4)

        login = self.client.login(username='staff', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('admin:download_sureal', args=(1,)))
        self.assertEqual(response.status_code, 200)
        with self.assertRaises(AssertionError):
            # ACR cannot do alt: ignore_against (there is no against to ignore)
            self.client.get(reverse('admin:download_sureal_alt', args=(1,)))

        sinfo = ec.get_session_info(sess)
        self.assertEqual(
            sinfo,
            {'session_id': 1, 'subject': 'user', 'rounds': [
                     {'round_id': 0, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 1, 'vote': 1.0}]},
                     {'round_id': 1, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0, 'vote': 4.0}]}]})

        # add second session with repetitions:
        sess2 = ec.add_session(subj)
        self.client.login(username='user', password='pass')
        self.client.get(reverse('nest:start_session', kwargs={'session_id': 2}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 2}), {'acr_0': '1'})
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 2}), {'acr_1': '3'})
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.login(username='staff', password='pass')

        sinfo2 = ec.get_session_info(sess2)
        self.assertEqual(
            sinfo2,
            {'session_id': 2, 'subject': 'user', 'rounds': [
                {'round_id': 0, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0, 'vote': 1.0}]},
                {'round_id': 1, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 1, 'vote': 3.0}]}]})

        einfo: dict = ec.get_experiment_info()
        df = ExperimentController.denormalize_experiment_info(einfo)
        self.assertEqual(df.shape, (4, 8))

        response = self.client.get(reverse('admin:download_sureal', args=(1,)))
        self.assertEqual(response.status_code, 200)
        dataset = export_sureal_dataset('nest_view_tests.TestViewsWithWriteDataset.test_step_session')
        self.assertEqual(dataset.dis_videos[0]['os'], {'user': [4.0, 1.0]})
        self.assertEqual(dataset.dis_videos[1]['os'], {'user': [1.0, 3.0]})

        # test reset session
        self.assertEqual(sess.round_set.first().vote_set.count(), 1)
        response = self.client.get(reverse('nest:reset_session', kwargs={'session_id': sess.id}))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(sess.round_set.first().vote_set.count(), 0)

    def test_step_session_acr5c(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_acr5c.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'acr5c_1': '18'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 18})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'acr5c_0': '89'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 89})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 0)
        self.assertEqual(Zero2HundredVote.objects.count(), 2)

        self.assertEqual(Vote.objects.first().score, 18)
        self.assertEqual(Vote.objects.all()[1].score, 89)
        self.assertEqual(Zero2HundredVote.objects.first().score, 18)
        self.assertEqual(Zero2HundredVote.objects.all()[1].score, 89)

    def test_step_session_acr5c_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_acr5c_standard.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'acr5c_1': '18'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 18})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'acr5c_0': '89'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 89})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 0)
        self.assertEqual(Zero2HundredVote.objects.count(), 2)

        self.assertEqual(Vote.objects.first().score, 18)
        self.assertEqual(Vote.objects.all()[1].score, 89)
        self.assertEqual(Zero2HundredVote.objects.first().score, 18)
        self.assertEqual(Zero2HundredVote.objects.all()[1].score, 89)

        login = self.client.login(username='staff', password='pass')
        self.assertTrue(login)

    def test_step_session_dcr(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_dcr.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_dcr')

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'dcr_1': '1'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 1})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'dcr_0': '5'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 5})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 5)
        self.assertEqual(FivePointVote.objects.first().score, 1)
        self.assertEqual(FivePointVote.objects.all()[1].score, 5)

        login = self.client.login(username='staff', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('admin:download_sureal', args=(1,)))
        self.assertEqual(response.status_code, 200)
        response = self.client.get(reverse('admin:download_sureal_alt', args=(1,)))
        self.assertEqual(response.status_code, 200)

        sinfo = ec.get_session_info(sess)
        self.assertEqual(
            sinfo,
            {'session_id': 1, 'subject': 'user', 'rounds': [
                {'round_id': 0, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 1, 'vote': 1.0}]},
                {'round_id': 1, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0, 'vote': 5.0}]}]})

        einfo: dict = ec.get_experiment_info()
        df = ExperimentController.denormalize_experiment_info(einfo)
        self.assertEqual(df.shape, (2, 10))

        r1 = Round.objects.get(session=sess, round_id=0)
        r2 = Round.objects.get(session=sess, round_id=1)
        self.assertTrue(r1.response_sec > 0)
        self.assertTrue(r2.response_sec > 0)

        # add second session with repetitions:
        sess2 = ec.add_session(subj)
        self.client.login(username='user', password='pass')
        self.client.get(reverse('nest:start_session', kwargs={'session_id': 2}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 2}), {'dcr_0': '1'})
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 2}), {'dcr_1': '4'})
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.login(username='staff', password='pass')

        sinfo2 = ec.get_session_info(sess2)
        self.assertEqual(
            sinfo2,
            {'session_id': 2, 'subject': 'user', 'rounds': [
                {'round_id': 0, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0, 'vote': 1.0}]},
                {'round_id': 1, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 1, 'vote': 4.0}]}]})

        response = self.client.get(reverse('admin:download_sureal', args=(1,)))
        self.assertEqual(response.status_code, 200)
        dataset = export_sureal_dataset('nest_view_tests.TestViewsWithWriteDataset.test_step_session_dcr',
                                        ignore_against=True)
        self.assertEqual(dataset.dis_videos[0]['os'], {'user': [5.0, 1.0]})
        self.assertEqual(dataset.dis_videos[1]['os'], {'user': [1.0, 4.0]})

    def test_step_session_dcr11d(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr11d.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_dcr11d')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'dcr_1': '1'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 1})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'dcr_0': '4'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 4})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(ElevenPointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 4)
        self.assertEqual(ElevenPointVote.objects.first().score, 1)
        self.assertEqual(ElevenPointVote.objects.all()[1].score, 4)

    def test_step_session_dcr3d_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr3d_standard.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_dcr3d_standard')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'dcr_1': '1'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 1})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'dcr_0': '3'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 3})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(ThreePointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 3)
        self.assertEqual(ThreePointVote.objects.first().score, 1)
        self.assertEqual(ThreePointVote.objects.all()[1].score, 3)

    def test_step_session_dcr7d_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr7d_standard.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_dcr7d_standard')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'dcr_1': '1'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 1})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'dcr_0': '7'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 7})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(SevenPointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 7)
        self.assertEqual(SevenPointVote.objects.first().score, 1)
        self.assertEqual(SevenPointVote.objects.all()[1].score, 7)

    def test_step_session_dcr11d_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_dcr11d_standard.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_dcr11d_standard')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'dcr_1': '1'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 1})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'dcr_0': '4'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 4})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(ElevenPointVote.objects.count(), 2)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 4)
        self.assertEqual(ElevenPointVote.objects.first().score, 1)
        self.assertEqual(ElevenPointVote.objects.all()[1].score, 4)

    def test_step_session_samviq(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_samviq.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_samviq')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}),
                                    {'samviq_2': "18", "samviq_3": "89"})  # key: svg; value: score
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'2': 18, '3': 89})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}),
                                    {"samviq_0": "2", "samviq_1": "4"})  # key: svg; value: score
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 2, '1': 4})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 4)
        self.assertEqual(FivePointVote.objects.count(), 0)
        self.assertEqual(Zero2HundredVote.objects.count(), 4)

        self.assertEqual(Vote.objects.first().score, 18)
        self.assertEqual(Vote.objects.all()[1].score, 89)
        self.assertEqual(Vote.objects.all()[2].score, 2)
        self.assertEqual(Vote.objects.all()[3].score, 4)

        stim_1_ref = Stimulus.objects.get(stimulus_id=0)
        stim_1_dis1 = Stimulus.objects.get(stimulus_id=1)
        stim_1_dis2 = Stimulus.objects.get(stimulus_id=2)
        stim_2_ref = Stimulus.objects.get(stimulus_id=3)
        stim_2_dis1 = Stimulus.objects.get(stimulus_id=4)
        stim_2_dis2 = Stimulus.objects.get(stimulus_id=5)
        svg_1_1 = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim_1_dis1, stim_1_ref)[0]
        svg_1_2 = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim_1_dis2, stim_1_ref)[0]
        svg_2_1 = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim_2_dis1, stim_2_ref)[0]
        svg_2_2 = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim_2_dis2, stim_2_ref)[0]
        self.assertEqual(Zero2HundredVote.objects.get(stimulusvotegroup=svg_1_1).score, 2)
        self.assertEqual(Zero2HundredVote.objects.get(stimulusvotegroup=svg_1_2).score, 4)
        self.assertEqual(Zero2HundredVote.objects.get(stimulusvotegroup=svg_2_1).score, 18)
        self.assertEqual(Zero2HundredVote.objects.get(stimulusvotegroup=svg_2_2).score, 89)

    def test_step_session_samviq5d(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_samviq5d.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_samviq5d')

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}),
                                    {'samviq5d_2': "1", "samviq5d_3": "3"})  # key: svg; value: score
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'2': 1, '3': 3})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}),
                                    {"samviq5d_0": "2", "samviq5d_1": "4"})  # key: svg; value: score
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 2, '1': 4})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 4)
        self.assertEqual(FivePointVote.objects.count(), 4)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 3)
        self.assertEqual(Vote.objects.all()[2].score, 2)
        self.assertEqual(Vote.objects.all()[3].score, 4)

        stim_1_ref = Stimulus.objects.get(stimulus_id=0)
        stim_1_dis1 = Stimulus.objects.get(stimulus_id=1)
        stim_1_dis2 = Stimulus.objects.get(stimulus_id=2)
        stim_2_ref = Stimulus.objects.get(stimulus_id=3)
        stim_2_dis1 = Stimulus.objects.get(stimulus_id=4)
        stim_2_dis2 = Stimulus.objects.get(stimulus_id=5)
        svg_1_1 = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim_1_dis1, stim_1_ref)[0]
        svg_1_2 = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim_1_dis2, stim_1_ref)[0]
        svg_2_1 = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim_2_dis1, stim_2_ref)[0]
        svg_2_2 = StimulusVoteGroup.find_stimulusvotegroups_from_stimuli_pair(stim_2_dis2, stim_2_ref)[0]
        self.assertEqual(FivePointVote.objects.get(stimulusvotegroup=svg_1_1).score, 2)
        self.assertEqual(FivePointVote.objects.get(stimulusvotegroup=svg_1_2).score, 4)
        self.assertEqual(FivePointVote.objects.get(stimulusvotegroup=svg_2_1).score, 1)
        self.assertEqual(FivePointVote.objects.get(stimulusvotegroup=svg_2_2).score, 3)

        sinfo = ec.get_session_info(sess)
        self.assertEqual(
            sinfo,
            {'session_id': 1, 'subject': 'user', 'rounds': [
                {'round_id': 0, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 2, 'vote': 1.0},
                                                                              {'stimulusvotegroup_id': 3, 'vote': 3.0}]},
                {'round_id': 1, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0, 'vote': 2.0},
                                                                              {'stimulusvotegroup_id': 1, 'vote': 4.0}]}]})

        einfo: dict = ec.get_experiment_info()
        df = ExperimentController.denormalize_experiment_info(einfo)
        self.assertEqual(df.shape, (4, 10))

        self.client.login(username='staff', password='pass')
        response = self.client.get(reverse('admin:download_sureal', args=(1,)))
        self.assertEqual(response.status_code, 200)
        response = self.client.get(reverse('admin:download_sureal_alt', args=(1,)))
        self.assertEqual(response.status_code, 200)
        dataset = export_sureal_dataset('nest_view_tests.TestViewsWithWriteDataset.test_step_session_samviq5d',
                                        ignore_against=True)
        self.assertEqual(dataset.dis_videos[0]['os'], {})
        self.assertEqual(dataset.dis_videos[1]['os'], {'user': 2.0})
        self.assertEqual(dataset.dis_videos[2]['os'], {'user': 4.0})
        self.assertEqual(dataset.dis_videos[3]['os'], {})
        self.assertEqual(dataset.dis_videos[4]['os'], {'user': 1.0})
        self.assertEqual(dataset.dis_videos[5]['os'], {'user': 3.0})

    def test_step_session_tafc(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_tafc.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_tafc')

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'ccr_1': '1'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 1})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'ccr_0': '0'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 0})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 0)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)
        self.assertEqual(TafcVote.objects.count(), 2)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 0)
        self.assertEqual(TafcVote.objects.first().score, 1)
        self.assertEqual(TafcVote.objects.all()[1].score, 0)

        login = self.client.login(username='staff', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('admin:download_sureal', args=(1,)))
        self.assertEqual(response.status_code, 200)
        response = self.client.get(reverse('admin:download_sureal_alt', args=(1,)))
        self.assertEqual(response.status_code, 200)

        sinfo = ec.get_session_info(sess)
        self.assertEqual(
            sinfo,
            {'session_id': 1, 'subject': 'user', 'rounds': [
                {'round_id': 0, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 1, 'vote': 1.0}]},
                {'round_id': 1, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0, 'vote': 0.0}]}]})

        # add second session with repetitions:
        sess2 = ec.add_session(subj)
        self.client.login(username='user', password='pass')
        self.client.get(reverse('nest:start_session', kwargs={'session_id': 2}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 2}), {'ccr_0': '1'})
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 2}), {'ccr_1': '0'})
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.login(username='staff', password='pass')

        sinfo2 = ec.get_session_info(sess2)
        self.assertEqual(
            sinfo2,
            {'session_id': 2, 'subject': 'user', 'rounds': [
                {'round_id': 0, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0, 'vote': 1.0}]},
                {'round_id': 1, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 1, 'vote': 0.0}]}]})

        response = self.client.get(reverse('admin:download_sureal', args=(1,)))
        self.assertEqual(response.status_code, 200)
        dataset = export_sureal_dataset('nest_view_tests.TestViewsWithWriteDataset.test_step_session_tafc',
                                        ignore_against=False)
        self.assertEqual(dataset.dis_videos[0]['os'], {('user', 0): [0.0, 1.0]})
        self.assertEqual(dataset.dis_videos[1]['os'], {('user', 0): [1.0, 0.0]})

        einfo: dict = ec.get_experiment_info()
        df = ExperimentController.denormalize_experiment_info(einfo)
        self.assertEqual(df.shape, (4, 10))

    def test_step_session_tafc_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_tafc_standard.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.cvxhull_subjexp_toy_x_tafc_standard')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'ccr_1': '1'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 1})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'ccr_0': '0'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 0})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 0)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)
        self.assertEqual(TafcVote.objects.count(), 2)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 0)
        self.assertEqual(TafcVote.objects.first().score, 1)
        self.assertEqual(TafcVote.objects.all()[1].score, 0)

    def test_step_session_mismatched_session_and_cache(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_tafc_standard.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.cvxhull_subjexp_toy_x_tafc_standard')

        ec2 = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_tafc.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_tafc')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)
        ec2.add_session(subj)

        self.client.login(username='user', password='pass')
        self.client.get(reverse('nest:status'))

        self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))

        with self.assertRaises(AssertionError):
            self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))

    def test_step_session_samviq_with_double_post_issue(self):
        # reproducing issue reported by TY 2/20/2022
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_samviq.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_samviq_with_double_post_issue')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        self.client.login(username='user', password='pass')
        self.client.get(reverse('nest:status'))
        self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))

        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))  # instruction

        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}),
                         {'samviq_2': "18", "samviq_3": "89"})  # round 1

        self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}),
                         {'samviq_2': "18", "samviq_3": "89"})  # round 1, double post

        # mistakenly submit, but should just be step:
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))

        # not yet recording result in database:
        self.assertEqual(Vote.objects.count(), 0)

    def test_step_session_samviq5d_with_double_post_issue(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_samviq5d.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_samviq5d_with_double_post_issue')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        self.client.login(username='user', password='pass')
        self.client.get(reverse('nest:status'))
        self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))

        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))  # instruction

        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}),
                         {'samviq5d_2': "1", "samviq5d_3": "3"})  # round 1

        self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}),
                         {'samviq5d_2': "1", "samviq5d_3": "3"})  # round 1, double post

        # mistakenly submit, but should just be step:
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))

        # not yet recording result in database:
        self.assertEqual(Vote.objects.count(), 0)

    def test_step_session_acr_with_double_post_issue(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_acr_with_double_post_issue')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        self.client.login(username='user', password='pass')
        self.client.get(reverse('nest:status'))
        self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))

        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))  # instruction

        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}),
                         {'acr_1': '1'})  # round 1

        self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}),
                         {'acr_1': '1'})  # round 1, double post

        # mistakenly submit, but should just be step:
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))

        # not yet recording result in database:
        self.assertEqual(Vote.objects.count(), 0)

    def test_step_session_ccr(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_ccr.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.test_step_session_ccr')

        subj: Subject = Subject.create_by_username('user')
        sess = ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'ccr_1': '2'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 2})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'ccr_0': '0'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 0})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 0)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)
        self.assertEqual(CcrThreePointVote.objects.count(), 2)

        self.assertEqual(Vote.objects.first().score, 2)
        self.assertEqual(Vote.objects.all()[1].score, 0)
        self.assertEqual(CcrThreePointVote.objects.first().score, 2)
        self.assertEqual(CcrThreePointVote.objects.all()[1].score, 0)

        login = self.client.login(username='staff', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('admin:download_sureal', args=(1,)))
        self.assertEqual(response.status_code, 200)
        response = self.client.get(reverse('admin:download_sureal_alt', args=(1,)))
        self.assertEqual(response.status_code, 200)

        sinfo = ec.get_session_info(sess)
        self.assertEqual(
            sinfo,
            {'session_id': 1, 'subject': 'user', 'rounds': [
                {'round_id': 0, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 1, 'vote': 2.0}]},
                {'round_id': 1, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0, 'vote': 0.0}]}]})

        # add second session with repetitions:
        sess2 = ec.add_session(subj)
        self.client.login(username='user', password='pass')
        self.client.get(reverse('nest:start_session', kwargs={'session_id': 2}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 2}), {'ccr_0': '2'})
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.post(reverse('nest:step_session', kwargs={'session_id': 2}), {'ccr_1': '0'})
        self.client.get(reverse('nest:step_session', kwargs={'session_id': 2}))
        self.client.login(username='staff', password='pass')

        sinfo2 = ec.get_session_info(sess2)
        self.assertEqual(
            sinfo2,
            {'session_id': 2, 'subject': 'user', 'rounds': [
                {'round_id': 0, 'stimulusgroup_id': 0, 'stimulusvotegroups': [{'stimulusvotegroup_id': 0, 'vote': 2.0}]},
                {'round_id': 1, 'stimulusgroup_id': 1, 'stimulusvotegroups': [{'stimulusvotegroup_id': 1, 'vote': 0.0}]}]})

        response = self.client.get(reverse('admin:download_sureal', args=(1,)))
        self.assertEqual(response.status_code, 200)
        dataset = export_sureal_dataset('nest_view_tests.TestViewsWithWriteDataset.test_step_session_ccr',
                                        ignore_against=False)
        self.assertEqual(dataset.dis_videos[0]['os'], {('user', 0): [0.0, 2.0]})
        self.assertEqual(dataset.dis_videos[1]['os'], {('user', 0): [2.0, 0.0]})

        einfo: dict = ec.get_experiment_info()
        df = ExperimentController.denormalize_experiment_info(einfo)
        self.assertEqual(df.shape, (4, 10))

    def test_step_session_ccr_standard(self):
        ec = ExperimentUtils._create_experiment_from_config(
            source_config_filepath=NestConfig.tests_resource_path('cvxhull_subjexp_toy_x_ccr_standard.json'),
            is_test=True,
            random_seed=1,
            experiment_title='nest_view_tests.TestViewsWithWriteDataset.cvxhull_subjexp_toy_x_ccr_standard')

        subj: Subject = Subject.create_by_username('user')
        ec.add_session(subj)

        login = self.client.login(username='user', password='pass')
        self.assertTrue(login)

        response = self.client.get(reverse('nest:status'))
        self.assertEqual(response.status_code, 200)

        response = self.client.get(reverse('nest:start_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 302)

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 1)
        self.assertTrue(NestSite._step_is_addition(steps[0]))

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'ccr_1': '1'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 2)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'1': 1})

        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('nest:step_session', kwargs={'session_id': 1}), {'ccr_0': '2'})
        self.assertEqual(response.status_code, 302)
        steps = self.client.session['steps']
        self.assertEqual(len(steps), 3)
        self.assertFalse(NestSite._step_is_addition(steps[-1]))
        self.assertEqual(steps[-1]['context']['score'], {'0': 2})

        self.assertEqual(Vote.objects.count(), 0)
        response = self.client.get(reverse('nest:step_session', kwargs={'session_id': 1}))
        self.assertEqual(response.status_code, 200)

        self.assertEqual(Vote.objects.count(), 2)
        self.assertEqual(FivePointVote.objects.count(), 0)
        self.assertEqual(Zero2HundredVote.objects.count(), 0)
        self.assertEqual(CcrThreePointVote.objects.count(), 2)

        self.assertEqual(Vote.objects.first().score, 1)
        self.assertEqual(Vote.objects.all()[1].score, 2)
        self.assertEqual(CcrThreePointVote.objects.first().score, 1)
        self.assertEqual(CcrThreePointVote.objects.all()[1].score, 2)

nest_site/nest/tests/nest_view_tests.py
Netflix/e2nest
import os

from django.test import TestCase
from django.urls import reverse_lazy
from nest.pages import Acr5cPage, AcrPage, CcrPage, DcrPage, GenericPage, \
    Samviq5dPage, SamviqPage

MEDIA_URL = ""


class TestPage(TestCase):

    def test_generic_page(self):
        title = 'Instruction'

        text_html = ""
        text_html += """ <p> In this session, you will be asked to evaluate the annoyance caused by presence of banding artifact in a set of videos. </p> """  # noqa E501
        text_html += """ <p> If you don't know what banding artifacts are, it would be good idea to contact the author <b>ptandon@catflix.com</b> before proceeding with the test. </p> """  # noqa E501
        text_html += """ <p> There are two phases, a training and a testing phase. In the training, you will see 6 videos and will be asked to rate the presence of banding artifacts. The videos are chosen to span the scale and there is an expected score for each of the video to provide an anchor for the testing videos. </p> """  # noqa E501
        text_html += """ <p> Next, the testing phase will start and at this point, your scores will be recorded.</b> </p> """  # noqa E501
        text_html += """ <p> To play a video, click on the <b>"Play Video"</b> button. </p> """  # noqa E501
        text_html += """ <p> <b style='color:red !important;'>Please, maintain the viewing distance of approximately 1.5 times the monitor height.</b> </p> """  # noqa E501
        text_html += """ <p> Video clips will loop over, <b>press "space" key or double click mouse </b> to terminate the video. A slider will appear and you will be able to rate annoyance caused by the presence of banding in the video on the 0-100 scale, qualitatively from unwatchable to imperceptible. Please try to focus only on the banding artifacts. For all clips rate the worst case banding seen during video playback, even for videos with multiple scenes. </p>  """  # noqa E501
        text_html += """ <p> Once you have scored the video, click the "Next" button to fetch next video and repeat. </p> """  # noqa E501
        text_html += """ <p> Progress is shown for training and testing cases on top, near instructions. </p> """  # noqa E501

        actions_html = \
            f"""
            <p>
                <a class="button" href="{reverse_lazy('nest:instruction_demo')}"  id="start">Start Evaluation</a>
            </p>
            """
        GenericPage({
            'title': title,
            'text_html': text_html,
            'actions_html': actions_html,
        })

    def test_acr_page(self):
        p = AcrPage({
            'title': "Round 1 of 10",
            'video': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_show_controls': True,
            'stimulusvotegroup_id': 0,
        })
        self.assertEqual(p.context['video_show_controls'], True)
        self.assertEqual(p.context['video_display_percentage'], 100)

    def test_acr_standard_page(self):
        p = AcrPage({
            'title': "Round 1 of 10",
            'video': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'template_version': 'standard',
            't_gray': 0,
            'num_plays': 2,
            'button': 'Watch the video',
            'stimulusvotegroup_id': 0,
        })
        self.assertEqual(p.context['video_show_controls'], False)
        self.assertEqual(p.context['video_display_percentage'], 100)

    def test_acr5c_page(self):
        p = Acr5cPage({
            'title': "Round 1 of 10",
            'video': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'stimulusvotegroup_id': 0,
        })
        self.assertEqual(p.context['video_show_controls'], False)
        self.assertEqual(p.context['video_display_percentage'], 100)

    def test_acr5c_standard_page(self):
        p = Acr5cPage({
            'title': "Round 1 of 10",
            'video': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'template_version': 'standard',
            't_gray': 0,
            'num_plays': 2,
            'button': 'Watch the video',
            'stimulusvotegroup_id': 0,
        })
        self.assertEqual(p.context['video_show_controls'], False)
        self.assertEqual(p.context['video_display_percentage'], 100)

    def test_dcr_page(self):
        p = DcrPage({
            'title': 'Round 1 of 10',
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'video_display_percentage': 50,
            'stimulusvotegroup_id': 0,
        })
        self.assertEqual(p.context['video_show_controls'], False)
        self.assertEqual(p.context['video_display_percentage'], 50)

    def test_ccr_page(self):
        p = CcrPage({
            'title': 'Round 1 of 10',
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'video_a_to_b_values': [0, 1, 2],
            'video_display_percentage': 100,
            'stimulusvotegroup_id': 0,
        })
        self.assertEqual(p.context['video_show_controls'], False)
        self.assertEqual(p.context['video_display_percentage'], 100)

        with self.assertRaises(AssertionError) as e:
            _ = CcrPage({
                'title': 'Round 1 of 10',
                'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
                'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                'video_a_to_b_values': [0, 1],
                'video_display_percentage': 100,
                'stimulusvotegroup_id': 0,
            })
        self.assertTrue('the length of choices and video_a_to_b_values must be equal' in e.exception.args[0])

    def test_ccr_standard_page(self):
        p = CcrPage({
            'title': 'Round 1 of 10',
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'video_a_to_b_values': [0, 1, 2],
            'template_version': "standard",
            'num_plays': 1,
            't_gray': 1000,
            'video_display_percentage': 70,
            'text_color': "#FFFFFF",
            'text_vert_perc': 45,
            'stimulusvotegroup_id': 0,
        })
        self.assertEqual(p.context['video_show_controls'], False)
        self.assertEqual(p.context['video_display_percentage'], 70)

        with self.assertRaises(AssertionError) as e:
            _ = CcrPage({
                'title': 'Round 1 of 10',
                'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
                'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                'video_a_to_b_values': [0, 1],
                'template_version': "standard",
                'num_plays': 1,
                't_gray': 1000,
                'video_display_percentage': 70,
                'text_color': "#FFFFFF",
                'text_vert_perc': 45,
                'stimulusvotegroup_id': 0,
            })
        self.assertTrue('the length of choices and video_a_to_b_values must be equal' in e.exception.args[0])

    def test_dcr11d_page(self):
        p = DcrPage({
            'title': 'Round 1 of 10',
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'choices':
                ['10 - Imperceptible',
                 '9 - Slightly perceptible somewhere',
                 '8 - Slightly perceptible everywhere',
                 '7 - Perceptible somewhere',
                 '6 - Perceptible everywhere',
                 '5 - Clearly perceptible somewhere',
                 '4 - Clearly perceptible everywhere',
                 '3 - Annoying somewhere',
                 '2 - Annoying everywhere',
                 '1 - Severely annoying somewhere',
                 '0 - Severely annoying everywhere'],
            'stimulusvotegroup_id': 0,
        })
        self.assertEqual(p.context['video_show_controls'], False)
        self.assertEqual(p.context['video_display_percentage'], 100)

    def test_dcr11d_standard_page(self):
        p = DcrPage({
            'title': 'Round 1 of 10',
            'video_a': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'video_b': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
            'template_version': "standard",
            'num_plays': 1,
            't_gray': 1000,
            'choices':
                ['10 - Imperceptible',
                 '9 - Slightly perceptible somewhere',
                 '8 - Slightly perceptible everywhere',
                 '7 - Perceptible somewhere',
                 '6 - Perceptible everywhere',
                 '5 - Clearly perceptible somewhere',
                 '4 - Clearly perceptible everywhere',
                 '3 - Annoying somewhere',
                 '2 - Annoying everywhere',
                 '1 - Severely annoying somewhere',
                 '0 - Severely annoying everywhere'],
            'stimulusvotegroup_id': 0,
        })
        self.assertEqual(p.context['video_show_controls'], False)
        self.assertEqual(p.context['video_display_percentage'], 100)

    def test_samviq_page(self):
        p = SamviqPage({
            'title': 'Round 1 of 10',
            'video_ref': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'button_ref': 'Reference',
            'videos': [os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       ],
            'buttons': ['A', 'B', 'C', 'D'],
            'stimulusvotegroup_ids': [0, 1, 2, 3],
        })
        self.assertEqual(p.context['video_show_controls'], False)
        self.assertEqual(p.context['video_display_percentage'], 100)

    def test_samviq5d_page(self):
        p = Samviq5dPage({
            'title': 'Round 1 of 10',
            'video_ref': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),  # noqa E501
            'button_ref': 'Reference',
            'videos': [os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),  # noqa E501
                       ],
            'buttons': ['A', 'B', 'C', 'D'],
            'stimulusvotegroup_ids': [0, 1, 2, 3],
        })
        self.assertEqual(p.context['video_show_controls'], False)
        self.assertEqual(p.context['video_display_percentage'], 100)

nest_site/nest/tests/page_tests.py
Netflix/e2nest
dataset_name = 'NFLX_public'
yuv_fmt = 'yuv420p'
width = 1920
height = 1080
ref_score = 5.0

ref_dir = '[path to dataset videos]/ref'
dis_dir = '[path to dataset videos]/dis'

ref_videos = [{'content_id': 0,
  'content_name': 'BigBuckBunny',
  'path': ref_dir + '/BigBuckBunny_25fps.yuv'},
 {'content_id': 1,
  'content_name': 'BirdsInCage',
  'path': ref_dir + '/BirdsInCage_30fps.yuv'},
 {'content_id': 2,
  'content_name': 'CrowdRun',
  'path': ref_dir + '/CrowdRun_25fps.yuv'}
]

dis_videos = [
 {'asset_id': 9,
  'content_id': 0,
  'os': [1.0,
         1.0,
         1.0,
         2.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         3.0,
         1.0,
         2.0,
         2.0,
         1.0,
         1.0,
         1.0,
         2.0,
         1.0,
         1.0,
         2.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         2.0,
         4.0,
         2.0,
         5.0,
         2.0],
  'path': dis_dir + '/BigBuckBunny_20_288_375.yuv',
 },
 {'asset_id': 10,
  'content_id': 0,
  'os': [2.0,
         1.0,
         2.0,
         3.0,
         3.0,
         1.0,
         2.0,
         2.0,
         2.0,
         3.0,
         2.0,
         3.0,
         4.0,
         2.0,
         1.0,
         2.0,
         2.0,
         1.0,
         2.0,
         2.0,
         3.0,
         1.0,
         1.0,
         2.0,
         3.0,
         2.0,
         4.0,
         2.0,
         1.0,
         1.0],
  'path': dis_dir + '/BigBuckBunny_30_384_550.yuv',
 },
 {'asset_id': 11,
  'content_id': 0,
  'os': [2.0,
         2.0,
         2.0,
         3.0,
         2.0,
         2.0,
         2.0,
         2.0,
         2.0,
         4.0,
         3.0,
         3.0,
         3.0,
         5.0,
         2.0,
         2.0,
         2.0,
         4.0,
         2.0,
         2.0,
         2.0,
         2.0,
         2.0,
         1.0,
         3.0,
         3.0,
         5.0,
         5.0,
         1.0,
         4.0],
  'path': dis_dir + '/BigBuckBunny_40_384_750.yuv',
 },
 {'asset_id': 12,
  'content_id': 0,
  'os': [3.0,
         3.0,
         4.0,
         4.0,
         4.0,
         3.0,
         3.0,
         3.0,
         3.0,
         5.0,
         3.0,
         4.0,
         4.0,
         3.0,
         2.0,
         2.0,
         3.0,
         3.0,
         3.0,
         3.0,
         3.0,
         2.0,
         3.0,
         2.0,
         4.0,
         3.0,
         4.0,
         3.0,
         3.0,
         2.0],
  'path': dis_dir + '/BigBuckBunny_50_480_1050.yuv',
 },
 {'asset_id': 13,
  'content_id': 0,
  'os': [4.0,
         3.0,
         5.0,
         4.0,
         4.0,
         4.0,
         4.0,
         4.0,
         4.0,
         5.0,
         4.0,
         4.0,
         5.0,
         4.0,
         3.0,
         3.0,
         4.0,
         3.0,
         3.0,
         4.0,
         3.0,
         3.0,
         4.0,
         2.0,
         4.0,
         5.0,
         5.0,
         4.0,
         5.0,
         1.0],
  'path': dis_dir + '/BigBuckBunny_55_480_1750.yuv',
 },
 {'asset_id': 14,
  'content_id': 0,
  'os': [5.0,
         3.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         4.0,
         5.0,
         4.0,
         5.0,
         5.0,
         3.0,
         5.0,
         4.0,
         4.0,
         5.0,
         5.0,
         5.0,
         3.0,
         5.0,
         4.0,
         4.0],
  'path': dis_dir + '/BigBuckBunny_75_720_3050.yuv',
 },
 {'asset_id': 15,
  'content_id': 0,
  'os': [5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         4.0,
         5.0,
         5.0,
         4.0,
         4.0,
         4.0,
         5.0,
         3.0,
         4.0,
         4.0,
         2.0,
         5.0,
         5.0,
         3.0],
  'path': dis_dir + '/BigBuckBunny_80_720_4250.yuv',
 },
 {'asset_id': 16,
  'content_id': 0,
  'os': [5.0,
         4.0,
         3.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         4.0,
         3.0,
         5.0,
         5.0,
         4.0,
         5.0,
         4.0,
         5.0],
  'path': dis_dir + '/BigBuckBunny_85_1080_3800.yuv',
 },
 {'asset_id': 17,
  'content_id': 0,
  'os': [5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         2.0,
         5.0,
         5.0],
  'path': dis_dir + '/BigBuckBunny_90_1080_4300.yuv',
 },
 {'asset_id': 18,
  'content_id': 0,
  'os': [5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         3.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         2.0,
         5.0,
         2.0],
  'path': dis_dir + '/BigBuckBunny_95_1080_5800.yuv',
 },
 {'asset_id': 0,
  'content_id': 0,
  'os': [5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         3.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         3.0,
         2.0,
         4.0],
  'path': ref_dir + '/BigBuckBunny_25fps.yuv',
 },
 {'asset_id': 19,
  'content_id': 1,
  'os': [2.0,
         1.0,
         3.0,
         3.0,
         2.0,
         2.0,
         2.0,
         3.0,
         1.0,
         3.0,
         3.0,
         2.0,
         3.0,
         5.0,
         1.0,
         1.0,
         2.0,
         2.0,
         2.0,
         2.0,
         1.0,
         2.0,
         1.0,
         1.0,
         3.0,
         2.0,
         2.0,
         3.0,
         4.0,
         3.0],
  'path': dis_dir + '/BirdsInCage_40_288_375.yuv',
 },
 {'asset_id': 20,
  'content_id': 1,
  'os': [1.0,
         1.0,
         3.0,
         3.0,
         2.0,
         1.0,
         2.0,
         2.0,
         1.0,
         3.0,
         2.0,
         2.0,
         4.0,
         2.0,
         3.0,
         1.0,
         2.0,
         2.0,
         1.0,
         3.0,
         2.0,
         2.0,
         1.0,
         1.0,
         2.0,
         2.0,
         5.0,
         4.0,
         5.0,
         4.0],
  'path': dis_dir + '/BirdsInCage_50_288_550.yuv',
 },
 {'asset_id': 21,
  'content_id': 1,
  'os': [2.0,
         2.0,
         4.0,
         3.0,
         1.0,
         3.0,
         2.0,
         3.0,
         3.0,
         4.0,
         2.0,
         3.0,
         4.0,
         3.0,
         3.0,
         1.0,
         2.0,
         2.0,
         2.0,
         3.0,
         3.0,
         3.0,
         2.0,
         2.0,
         3.0,
         4.0,
         4.0,
         4.0,
         5.0,
         2.0],
  'path': dis_dir + '/BirdsInCage_60_384_550.yuv',
 },
 {'asset_id': 22,
  'content_id': 1,
  'os': [2.0,
         2.0,
         4.0,
         3.0,
         3.0,
         2.0,
         3.0,
         3.0,
         3.0,
         5.0,
         3.0,
         3.0,
         4.0,
         2.0,
         3.0,
         2.0,
         3.0,
         3.0,
         2.0,
         3.0,
         2.0,
         3.0,
         2.0,
         2.0,
         4.0,
         2.0,
         4.0,
         5.0,
         2.0,
         3.0],
  'path': dis_dir + '/BirdsInCage_65_384_750.yuv',
 },
 {'asset_id': 23,
  'content_id': 1,
  'os': [3.0,
         3.0,
         4.0,
         4.0,
         4.0,
         4.0,
         4.0,
         3.0,
         3.0,
         5.0,
         3.0,
         4.0,
         4.0,
         4.0,
         4.0,
         3.0,
         4.0,
         3.0,
         3.0,
         4.0,
         3.0,
         3.0,
         4.0,
         3.0,
         4.0,
         3.0,
         4.0,
         1.0,
         2.0,
         5.0],
  'path': dis_dir + '/BirdsInCage_80_480_750.yuv',
 },
 {'asset_id': 24,
  'content_id': 1,
  'os': [4.0,
         4.0,
         5.0,
         5.0,
         4.0,
         5.0,
         4.0,
         4.0,
         4.0,
         5.0,
         5.0,
         4.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         3.0,
         4.0,
         4.0,
         5.0,
         5.0,
         4.0,
         4.0,
         5.0,
         4.0,
         4.0,
         4.0,
         4.0,
         5.0],
  'path': dis_dir + '/BirdsInCage_85_720_1050.yuv',
 },
 {'asset_id': 25,
  'content_id': 1,
  'os': [5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         1.0,
         5.0,
         5.0,
         4.0],
  'path': dis_dir + '/BirdsInCage_90_1080_1800.yuv',
 },
 {'asset_id': 26,
  'content_id': 1,
  'os': [4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         4.0],
  'path': dis_dir + '/BirdsInCage_95_1080_3000.yuv',
 },
 {'asset_id': 1,
  'content_id': 1,
  'os': [4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         3.0,
         5.0,
         1.0,
         5.0],
  'path': ref_dir + '/BirdsInCage_30fps.yuv',
 },
 {'asset_id': 27,
  'content_id': 2,
  'os': [1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         1.0,
         5.0,
         2.0,
         4.0,
         5.0],
  'path': dis_dir + '/CrowdRun_03_288_375.yuv',
 },
 {'asset_id': 28,
  'content_id': 2,
  'os': [2.0,
         2.0,
         2.0,
         2.0,
         2.0,
         1.0,
         2.0,
         3.0,
         1.0,
         3.0,
         1.0,
         3.0,
         1.0,
         2.0,
         2.0,
         2.0,
         2.0,
         2.0,
         2.0,
         2.0,
         1.0,
         2.0,
         2.0,
         1.0,
         3.0,
         2.0,
         4.0,
         3.0,
         4.0,
         5.0],
  'path': dis_dir + '/CrowdRun_40_480_2350.yuv',
 },
 {'asset_id': 29,
  'content_id': 2,
  'os': [2.0,
         3.0,
         3.0,
         5.0,
         3.0,
         3.0,
         1.0,
         3.0,
         1.0,
         4.0,
         3.0,
         4.0,
         3.0,
         4.0,
         3.0,
         3.0,
         3.0,
         2.0,
         1.0,
         2.0,
         2.0,
         3.0,
         3.0,
         2.0,
         3.0,
         3.0,
         5.0,
         5.0,
         4.0,
         2.0],
  'path': dis_dir + '/CrowdRun_50_1080_4300.yuv',
 },
 {'asset_id': 30,
  'content_id': 2,
  'os': [3.0,
         3.0,
         4.0,
         3.0,
         4.0,
         3.0,
         3.0,
         4.0,
         3.0,
         4.0,
         4.0,
         4.0,
         4.0,
         3.0,
         3.0,
         3.0,
         3.0,
         2.0,
         2.0,
         3.0,
         4.0,
         3.0,
         4.0,
         2.0,
         4.0,
         3.0,
         5.0,
         4.0,
         2.0,
         3.0],
  'path': dis_dir + '/CrowdRun_65_1080_5800.yuv',
 },
 {'asset_id': 31,
  'content_id': 2,
  'os': [3.0,
         3.0,
         5.0,
         5.0,
         4.0,
         5.0,
         3.0,
         5.0,
         4.0,
         5.0,
         4.0,
         4.0,
         4.0,
         4.0,
         4.0,
         5.0,
         4.0,
         3.0,
         2.0,
         4.0,
         4.0,
         4.0,
         3.0,
         4.0,
         5.0,
         3.0,
         4.0,
         3.0,
         2.0,
         4.0],
  'path': dis_dir + '/CrowdRun_75_1080_7500.yuv',
 },
 {'asset_id': 32,
  'content_id': 2,
  'os': [5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         3.0,
         5.0,
         5.0,
         4.0,
         5.0,
         4.0,
         5.0,
         4.0,
         5.0,
         4.0,
         5.0,
         4.0,
         4.0,
         4.0,
         3.0,
         4.0,
         4.0,
         4.0,
         4.0,
         5.0,
         4.0,
         2.0,
         3.0,
         1.0,
         1.0],
  'path': dis_dir + '/CrowdRun_80_1080_10000.yuv',
 },
 {'asset_id': 33,
  'content_id': 2,
  'os': [5.0,
         5.0,
         4.0,
         4.0,
         5.0,
         4.0,
         4.0,
         5.0,
         4.0,
         5.0,
         4.0,
         4.0,
         4.0,
         4.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         4.0,
         5.0,
         4.0,
         5.0,
         3.0,
         4.0,
         2.0],
  'path': dis_dir + '/CrowdRun_90_1080_15000.yuv',
 },
 {'asset_id': 2,
  'content_id': 2,
  'os': [5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         3.0,
         4.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         4.0,
         4.0,
         5.0,
         5.0,
         5.0,
         4.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         5.0,
         1.0],
  'path': ref_dir + '/CrowdRun_25fps.yuv',
 },
]
nest_site/nest/tests/resource/NFLX_dataset_public_raw_last4outliers_tiny.py
Netflix/e2nest
import os

from nest_site.settings import MEDIA_URL


dataset_name = 'cvxhull_subjexp'
yuv_fmt = 'notyuv'
quality_width = 1920
quality_height = 1080

ref_videos = [{'content_id': 0,
  'content_name': 'CTS3E1_B',
  'path': os.path.join(MEDIA_URL, "mp4/samples/Meridian/Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4")},
]

dis_videos = [
 {'asset_id': 0,
  'content_id': 0,
  'os': [],
  'path': os.path.join(MEDIA_URL, "mp4", "samples", "Meridian", "Meridian_A__8_18_8_23__SdrVvhevce2pVE__3840_2160__6000_enable_audio_False_vmaf103.58_phonevmaf104.85_psnr50.40_kbps6702.77.mp4"),
 },
 {'asset_id': 1,
  'content_id': 0,
  'os': [],
  'path': os.path.join(MEDIA_URL, "mp4", "samples", "Meridian", "Meridian_A__8_18_8_23__SdrVvhevce2pVE__960_540__500_enable_audio_False_vmaf87.25_phonevmaf98.62_psnr45.35_kbps559.20.mp4"),
 },
]

nest_site/nest/tests/resource/cvxhull_subjexp_toy.py
Netflix/e2nest
# Dataset taken from:
# Quality assessment of sharpened images: challenges, methodology, and objective metrics
# L Krasula, P Le Callet, K Fliegel, M Klima - IEEE Transactions on Image Processing, 2017
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

dataset_name = 'Lukas_SPIE14'
yuv_fmt = 'notyuv'
quality_width = 512
quality_height = 512
ref_score = 0

ref_videos = [{'content_id': 0,
  'content_name': 'Caps',
  'path': 'Caps1.png'},
 {'content_id': 1,
  'content_name': 'parrots',
  'path': 'parrots1.png'},
]

dis_videos = [{'asset_id': 0,
  'content_id': 0,
  'os': {('David R.', 2): 1,
         ('David R.', 3): 1,
         ('David R.', 4): 1,
         ('David R.', 5): 1,
         ('David R.', 6): 1,
         ('David R.', 7): 1,
         ('Fritol', 1): 1,
         ('Fritol', 3): 1,
         ('Fritol', 4): 1,
         ('Fritol', 5): 1,
         ('Fritol', 6): 1,
         ('Fritol', 7): 1,
         ('Jana', 6): 1,
         ('Jana', 7): 1,
         ('Josef', 6): 1,
         ('Josef', 7): 1,
         ('Martin Vitvar', 3): 1,
         ('Martin Vitvar', 4): 1,
         ('Martin Vitvar', 5): 1,
         ('Martin Vitvar', 6): 1,
         ('Martin Vitvar', 7): 1,
         ('Petr Hl', 3): 1,
         ('Petr Hl', 5): 1,
         ('Petr Hl', 6): 1,
         ('Petr Hl', 7): 1,
         ('Voldamic', 3): 1,
         ('Voldamic', 5): 1,
         ('Voldamic', 6): 1,
         ('Voldamic', 7): 1,
         ('budulinek', 3): 1,
         ('budulinek', 4): 1,
         ('budulinek', 5): 1,
         ('budulinek', 6): 1,
         ('budulinek', 7): 1,
         ('delcamat', 1): 1,
         ('delcamat', 3): 1,
         ('delcamat', 4): 1,
         ('delcamat', 5): 1,
         ('delcamat', 7): 1,
         ('dingo', 3): 1,
         ('dingo', 5): 1,
         ('dingo', 6): 1,
         ('dingo', 7): 1,
         ('dominik', 3): 1,
         ('dominik', 5): 1,
         ('dominik', 6): 1,
         ('dominik', 7): 1,
         ('eleanor', 1): 1,
         ('eleanor', 2): 1,
         ('eleanor', 3): 1,
         ('eleanor', 4): 1,
         ('eleanor', 5): 1,
         ('eleanor', 6): 1,
         ('eleanor', 7): 1,
         ('lukas', 1): 1,
         ('lukas', 3): 1,
         ('lukas', 4): 1,
         ('lukas', 5): 1,
         ('lukas', 6): 1,
         ('lukas', 7): 1,
         ('madrvojt', 3): 1,
         ('madrvojt', 4): 1,
         ('madrvojt', 5): 1,
         ('madrvojt', 6): 1,
         ('madrvojt', 7): 1},
  'path': 'Caps1.png'},
 {'asset_id': 1,
  'content_id': 0,
  'os': {('David R.', 0): 1,
         ('David R.', 2): 1,
         ('David R.', 4): 1,
         ('David R.', 5): 1,
         ('David R.', 6): 1,
         ('David R.', 7): 1,
         ('Fritol', 2): 1,
         ('Fritol', 3): 1,
         ('Fritol', 4): 1,
         ('Fritol', 5): 1,
         ('Fritol', 6): 1,
         ('Fritol', 7): 1,
         ('Jana', 0): 1,
         ('Jana', 3): 1,
         ('Jana', 4): 1,
         ('Jana', 5): 1,
         ('Jana', 6): 1,
         ('Jana', 7): 1,
         ('Josef', 0): 1,
         ('Josef', 3): 1,
         ('Josef', 4): 1,
         ('Josef', 5): 1,
         ('Josef', 6): 1,
         ('Josef', 7): 1,
         ('Martin Vitvar', 0): 1,
         ('Martin Vitvar', 2): 1,
         ('Martin Vitvar', 3): 1,
         ('Martin Vitvar', 4): 1,
         ('Martin Vitvar', 5): 1,
         ('Martin Vitvar', 6): 1,
         ('Martin Vitvar', 7): 1,
         ('Petr Hl', 0): 1,
         ('Petr Hl', 2): 1,
         ('Petr Hl', 4): 1,
         ('Petr Hl', 5): 1,
         ('Petr Hl', 6): 1,
         ('Petr Hl', 7): 1,
         ('Voldamic', 0): 1,
         ('Voldamic', 2): 1,
         ('Voldamic', 3): 1,
         ('Voldamic', 4): 1,
         ('Voldamic', 5): 1,
         ('budulinek', 0): 1,
         ('budulinek', 3): 1,
         ('budulinek', 4): 1,
         ('budulinek', 5): 1,
         ('budulinek', 6): 1,
         ('budulinek', 7): 1,
         ('cinklond', 0): 1,
         ('delcamat', 2): 1,
         ('delcamat', 3): 1,
         ('delcamat', 4): 1,
         ('delcamat', 5): 1,
         ('delcamat', 6): 1,
         ('delcamat', 7): 1,
         ('dingo', 0): 1,
         ('dingo', 2): 1,
         ('dingo', 3): 1,
         ('dingo', 4): 1,
         ('dingo', 5): 1,
         ('dingo', 6): 1,
         ('dingo', 7): 1,
         ('dominik', 0): 1,
         ('dominik', 3): 1,
         ('dominik', 4): 1,
         ('dominik', 5): 1,
         ('dominik', 6): 1,
         ('dominik', 7): 1,
         ('eleanor', 2): 1,
         ('eleanor', 3): 1,
         ('eleanor', 4): 1,
         ('eleanor', 5): 1,
         ('eleanor', 6): 1,
         ('eleanor', 7): 1,
         ('lukas', 2): 1,
         ('lukas', 3): 1,
         ('lukas', 4): 1,
         ('lukas', 5): 1,
         ('lukas', 6): 1,
         ('lukas', 7): 1,
         ('madrvojt', 0): 1,
         ('madrvojt', 3): 1,
         ('madrvojt', 4): 1,
         ('madrvojt', 5): 1,
         ('madrvojt', 6): 1,
         ('madrvojt', 7): 1},
  'path': 'Caps2.png'},
 {'asset_id': 2,
  'content_id': 0,
  'os': {('David R.', 4): 1,
         ('David R.', 5): 1,
         ('David R.', 6): 1,
         ('David R.', 7): 1,
         ('Fritol', 0): 1,
         ('Fritol', 3): 1,
         ('Fritol', 4): 1,
         ('Fritol', 5): 1,
         ('Fritol', 6): 1,
         ('Fritol', 7): 1,
         ('Jana', 0): 1,
         ('Jana', 1): 1,
         ('Jana', 3): 1,
         ('Jana', 4): 1,
         ('Jana', 6): 1,
         ('Jana', 7): 1,
         ('Josef', 0): 1,
         ('Josef', 1): 1,
         ('Josef', 4): 1,
         ('Josef', 5): 1,
         ('Josef', 6): 1,
         ('Josef', 7): 1,
         ('Martin Vitvar', 0): 1,
         ('Martin Vitvar', 3): 1,
         ('Martin Vitvar', 4): 1,
         ('Martin Vitvar', 5): 1,
         ('Martin Vitvar', 6): 1,
         ('Martin Vitvar', 7): 1,
         ('Petr Hl', 0): 1,
         ('Petr Hl', 3): 1,
         ('Petr Hl', 4): 1,
         ('Petr Hl', 5): 1,
         ('Petr Hl', 6): 1,
         ('Petr Hl', 7): 1,
         ('Voldamic', 0): 1,
         ('Voldamic', 3): 1,
         ('Voldamic', 5): 1,
         ('Voldamic', 6): 1,
         ('Voldamic', 7): 1,
         ('budulinek', 0): 1,
         ('budulinek', 1): 1,
         ('budulinek', 3): 1,
         ('budulinek', 6): 1,
         ('budulinek', 7): 1,
         ('cinklond', 0): 1,
         ('cinklond', 1): 1,
         ('cinklond', 6): 1,
         ('cinklond', 7): 1,
         ('delcamat', 0): 1,
         ('delcamat', 3): 1,
         ('delcamat', 4): 1,
         ('delcamat', 5): 1,
         ('delcamat', 6): 1,
         ('delcamat', 7): 1,
         ('dingo', 0): 1,
         ('dingo', 3): 1,
         ('dingo', 4): 1,
         ('dingo', 5): 1,
         ('dingo', 6): 1,
         ('dingo', 7): 1,
         ('dominik', 0): 1,
         ('dominik', 1): 1,
         ('dominik', 4): 1,
         ('dominik', 5): 1,
         ('dominik', 6): 1,
         ('dominik', 7): 1,
         ('eleanor', 3): 1,
         ('eleanor', 4): 1,
         ('eleanor', 5): 1,
         ('eleanor', 6): 1,
         ('eleanor', 7): 1,
         ('lukas', 0): 1,
         ('lukas', 3): 1,
         ('lukas', 4): 1,
         ('lukas', 5): 1,
         ('lukas', 6): 1,
         ('lukas', 7): 1,
         ('madrvojt', 0): 1,
         ('madrvojt', 1): 1,
         ('madrvojt', 5): 1,
         ('madrvojt', 6): 1,
         ('madrvojt', 7): 1},
  'path': 'Caps3.png'},
 {'asset_id': 3,
  'content_id': 0,
  'os': {('David R.', 1): 1,
         ('David R.', 2): 1,
         ('David R.', 4): 1,
         ('David R.', 5): 1,
         ('David R.', 6): 1,
         ('David R.', 7): 1,
         ('Fritol', 5): 1,
         ('Fritol', 6): 1,
         ('Fritol', 7): 1,
         ('Jana', 0): 1,
         ('Jana', 6): 1,
         ('Jana', 7): 1,
         ('Josef', 0): 1,
         ('Josef', 2): 1,
         ('Josef', 4): 1,
         ('Josef', 5): 1,
         ('Josef', 6): 1,
         ('Josef', 7): 1,
         ('Martin Vitvar', 4): 1,
         ('Martin Vitvar', 5): 1,
         ('Martin Vitvar', 6): 1,
         ('Martin Vitvar', 7): 1,
         ('Petr Hl', 1): 1,
         ('Petr Hl', 4): 1,
         ('Petr Hl', 5): 1,
         ('Petr Hl', 6): 1,
         ('Petr Hl', 7): 1,
         ('Voldamic', 7): 1,
         ('budulinek', 5): 1,
         ('budulinek', 6): 1,
         ('budulinek', 7): 1,
         ('cinklond', 0): 1,
         ('cinklond', 1): 1,
         ('cinklond', 2): 1,
         ('cinklond', 6): 1,
         ('delcamat', 4): 1,
         ('delcamat', 5): 1,
         ('delcamat', 6): 1,
         ('delcamat', 7): 1,
         ('dingo', 4): 1,
         ('dingo', 5): 1,
         ('dingo', 6): 1,
         ('dingo', 7): 1,
         ('dominik', 2): 1,
         ('dominik', 4): 1,
         ('dominik', 5): 1,
         ('dominik', 6): 1,
         ('dominik', 7): 1,
         ('eleanor', 4): 1,
         ('eleanor', 5): 1,
         ('eleanor', 6): 1,
         ('eleanor', 7): 1,
         ('lukas', 4): 1,
         ('lukas', 5): 1,
         ('lukas', 6): 1,
         ('lukas', 7): 1,
         ('madrvojt', 2): 1,
         ('madrvojt', 4): 1,
         ('madrvojt', 5): 1,
         ('madrvojt', 6): 1,
         ('madrvojt', 7): 1},
  'path': 'Caps4.png'},
 {'asset_id': 4,
  'content_id': 0,
  'os': {('David R.', 5): 1,
         ('David R.', 6): 1,
         ('David R.', 7): 1,
         ('Fritol', 3): 1,
         ('Fritol', 5): 1,
         ('Fritol', 6): 1,
         ('Fritol', 7): 1,
         ('Jana', 0): 1,
         ('Jana', 3): 1,
         ('Jana', 6): 1,
         ('Jana', 7): 1,
         ('Josef', 0): 1,
         ('Josef', 6): 1,
         ('Josef', 7): 1,
         ('Martin Vitvar', 5): 1,
         ('Martin Vitvar', 6): 1,
         ('Martin Vitvar', 7): 1,
         ('Petr Hl', 0): 1,
         ('Petr Hl', 5): 1,
         ('Petr Hl', 6): 1,
         ('Petr Hl', 7): 1,
         ('Voldamic', 0): 1,
         ('Voldamic', 2): 1,
         ('Voldamic', 3): 1,
         ('Voldamic', 7): 1,
         ('budulinek', 2): 1,
         ('budulinek', 3): 1,
         ('budulinek', 6): 1,
         ('budulinek', 7): 1,
         ('cinklond', 0): 1,
         ('cinklond', 1): 1,
         ('cinklond', 2): 1,
         ('cinklond', 3): 1,
         ('cinklond', 5): 1,
         ('cinklond', 6): 1,
         ('cinklond', 7): 1,
         ('delcamat', 7): 1,
         ('dingo', 0): 1,
         ('dingo', 5): 1,
         ('dingo', 6): 1,
         ('dingo', 7): 1,
         ('dominik', 0): 1,
         ('dominik', 5): 1,
         ('dominik', 6): 1,
         ('dominik', 7): 1,
         ('eleanor', 6): 1,
         ('eleanor', 7): 1,
         ('lukas', 5): 1,
         ('lukas', 6): 1,
         ('lukas', 7): 1,
         ('madrvojt', 2): 1,
         ('madrvojt', 5): 1,
         ('madrvojt', 6): 1,
         ('madrvojt', 7): 1},
  'path': 'Caps5.png'},
 {'asset_id': 5,
  'content_id': 0,
  'os': {('David R.', 6): 1,
         ('David R.', 7): 1,
         ('Fritol', 6): 1,
         ('Jana', 0): 1,
         ('Jana', 2): 1,
         ('Jana', 3): 1,
         ('Jana', 4): 1,
         ('Jana', 7): 1,
         ('Josef', 0): 1,
         ('Josef', 4): 1,
         ('Josef', 6): 1,
         ('Josef', 7): 1,
         ('Martin Vitvar', 6): 1,
         ('Martin Vitvar', 7): 1,
         ('Petr Hl', 6): 1,
         ('Petr Hl', 7): 1,
         ('Voldamic', 3): 1,
         ('Voldamic', 4): 1,
         ('Voldamic', 6): 1,
         ('Voldamic', 7): 1,
         ('budulinek', 2): 1,
         ('budulinek', 4): 1,
         ('budulinek', 6): 1,
         ('budulinek', 7): 1,
         ('cinklond', 0): 1,
         ('cinklond', 1): 1,
         ('cinklond', 2): 1,
         ('cinklond', 3): 1,
         ('cinklond', 7): 1,
         ('delcamat', 4): 1,
         ('delcamat', 7): 1,
         ('dingo', 6): 1,
         ('dingo', 7): 1,
         ('dominik', 6): 1,
         ('dominik', 7): 1,
         ('eleanor', 4): 1,
         ('eleanor', 6): 1,
         ('eleanor', 7): 1,
         ('lukas', 7): 1,
         ('madrvojt', 7): 1},
  'path': 'Caps6.png'},
 {'asset_id': 6,
  'content_id': 0,
  'os': {('David R.', 7): 1,
         ('Fritol', 7): 1,
         ('Jana', 5): 1,
         ('Josef', 7): 1,
         ('Martin Vitvar', 7): 1,
         ('Petr Hl', 7): 1,
         ('Voldamic', 1): 1,
         ('Voldamic', 3): 1,
         ('Voldamic', 4): 1,
         ('Voldamic', 7): 1,
         ('budulinek', 7): 1,
         ('cinklond', 0): 1,
         ('cinklond', 1): 1,
         ('cinklond', 5): 1,
         ('delcamat', 0): 1,
         ('delcamat', 4): 1,
         ('delcamat', 5): 1,
         ('dingo', 7): 1,
         ('lukas', 5): 1,
         ('lukas', 7): 1,
         ('madrvojt', 5): 1,
         ('madrvojt', 7): 1},
  'path': 'Caps7.png'},
 {'asset_id': 7,
  'content_id': 0,
  'os': {('Fritol', 5): 1,
         ('Jana', 6): 1,
         ('Voldamic', 1): 1,
         ('cinklond', 0): 1,
         ('cinklond', 1): 1,
         ('cinklond', 3): 1,
         ('cinklond', 6): 1,
         ('delcamat', 6): 1,
         ('dominik', 6): 1,
         ('eleanor', 6): 1},
  'path': 'Caps8.png'},
 {'asset_id': 8,
  'content_id': 1,
  'os': {('David R.', 10): 1,
         ('David R.', 11): 1,
         ('David R.', 13): 1,
         ('David R.', 14): 1,
         ('David R.', 15): 1,
         ('Fritol', 14): 1,
         ('Fritol', 15): 1,
         ('Jana', 10): 1,
         ('Jana', 12): 1,
         ('Jana', 13): 1,
         ('Jana', 14): 1,
         ('Jana', 15): 1,
         ('Josef', 11): 1,
         ('Josef', 12): 1,
         ('Josef', 13): 1,
         ('Josef', 14): 1,
         ('Josef', 15): 1,
         ('Martin Vitvar', 11): 1,
         ('Martin Vitvar', 12): 1,
         ('Martin Vitvar', 13): 1,
         ('Martin Vitvar', 14): 1,
         ('Martin Vitvar', 15): 1,
         ('Petr Hl', 12): 1,
         ('Petr Hl', 14): 1,
         ('Petr Hl', 15): 1,
         ('Voldamic', 10): 1,
         ('Voldamic', 11): 1,
         ('Voldamic', 12): 1,
         ('Voldamic', 13): 1,
         ('Voldamic', 14): 1,
         ('Voldamic', 15): 1,
         ('budulinek', 9): 1,
         ('budulinek', 11): 1,
         ('budulinek', 12): 1,
         ('budulinek', 13): 1,
         ('budulinek', 14): 1,
         ('budulinek', 15): 1,
         ('cinklond', 13): 1,
         ('cinklond', 14): 1,
         ('cinklond', 15): 1,
         ('delcamat', 9): 1,
         ('delcamat', 10): 1,
         ('delcamat', 12): 1,
         ('delcamat', 13): 1,
         ('delcamat', 14): 1,
         ('delcamat', 15): 1,
         ('dingo', 9): 1,
         ('dingo', 10): 1,
         ('dingo', 11): 1,
         ('dingo', 12): 1,
         ('dingo', 13): 1,
         ('dingo', 14): 1,
         ('dingo', 15): 1,
         ('dominik', 12): 1,
         ('dominik', 13): 1,
         ('dominik', 14): 1,
         ('dominik', 15): 1,
         ('eleanor', 9): 1,
         ('eleanor', 10): 1,
         ('eleanor', 11): 1,
         ('eleanor', 12): 1,
         ('eleanor', 13): 1,
         ('eleanor', 14): 1,
         ('eleanor', 15): 1,
         ('lukas', 9): 1,
         ('lukas', 10): 1,
         ('lukas', 11): 1,
         ('lukas', 12): 1,
         ('lukas', 13): 1,
         ('lukas', 14): 1,
         ('lukas', 15): 1,
         ('madrvojt', 9): 1,
         ('madrvojt', 11): 1,
         ('madrvojt', 12): 1,
         ('madrvojt', 13): 1,
         ('madrvojt', 14): 1,
         ('madrvojt', 15): 1},
  'path': 'parrots1.png'},
 {'asset_id': 9,
  'content_id': 1,
  'os': {('David R.', 8): 1,
         ('David R.', 10): 1,
         ('David R.', 11): 1,
         ('David R.', 12): 1,
         ('David R.', 13): 1,
         ('David R.', 14): 1,
         ('David R.', 15): 1,
         ('Fritol', 8): 1,
         ('Fritol', 11): 1,
         ('Fritol', 12): 1,
         ('Fritol', 13): 1,
         ('Fritol', 14): 1,
         ('Jana', 8): 1,
         ('Jana', 11): 1,
         ('Jana', 12): 1,
         ('Jana', 13): 1,
         ('Jana', 14): 1,
         ('Jana', 15): 1,
         ('Josef', 8): 1,
         ('Josef', 11): 1,
         ('Josef', 12): 1,
         ('Josef', 13): 1,
         ('Josef', 14): 1,
         ('Josef', 15): 1,
         ('Martin Vitvar', 8): 1,
         ('Martin Vitvar', 11): 1,
         ('Martin Vitvar', 12): 1,
         ('Martin Vitvar', 13): 1,
         ('Martin Vitvar', 14): 1,
         ('Martin Vitvar', 15): 1,
         ('Petr Hl', 8): 1,
         ('Petr Hl', 12): 1,
         ('Petr Hl', 13): 1,
         ('Petr Hl', 14): 1,
         ('Petr Hl', 15): 1,
         ('Voldamic', 8): 1,
         ('Voldamic', 10): 1,
         ('Voldamic', 11): 1,
         ('Voldamic', 12): 1,
         ('Voldamic', 14): 1,
         ('Voldamic', 15): 1,
         ('budulinek', 10): 1,
         ('budulinek', 11): 1,
         ('budulinek', 12): 1,
         ('budulinek', 13): 1,
         ('budulinek', 14): 1,
         ('budulinek', 15): 1,
         ('cinklond', 8): 1,
         ('cinklond', 10): 1,
         ('cinklond', 11): 1,
         ('cinklond', 12): 1,
         ('cinklond', 13): 1,
         ('cinklond', 14): 1,
         ('cinklond', 15): 1,
         ('delcamat', 10): 1,
         ('delcamat', 11): 1,
         ('delcamat', 12): 1,
         ('delcamat', 13): 1,
         ('delcamat', 14): 1,
         ('delcamat', 15): 1,
         ('dingo', 11): 1,
         ('dingo', 12): 1,
         ('dingo', 13): 1,
         ('dingo', 14): 1,
         ('dingo', 15): 1,
         ('dominik', 8): 1,
         ('dominik', 10): 1,
         ('dominik', 11): 1,
         ('dominik', 12): 1,
         ('dominik', 13): 1,
         ('dominik', 14): 1,
         ('dominik', 15): 1,
         ('eleanor', 10): 1,
         ('eleanor', 11): 1,
         ('eleanor', 12): 1,
         ('eleanor', 13): 1,
         ('eleanor', 14): 1,
         ('eleanor', 15): 1,
         ('lukas', 10): 1,
         ('lukas', 11): 1,
         ('lukas', 12): 1,
         ('lukas', 13): 1,
         ('lukas', 14): 1,
         ('lukas', 15): 1,
         ('madrvojt', 10): 1,
         ('madrvojt', 11): 1,
         ('madrvojt', 12): 1,
         ('madrvojt', 13): 1,
         ('madrvojt', 14): 1,
         ('madrvojt', 15): 1},
  'path': 'parrots2.png'},
 {'asset_id': 10,
  'content_id': 1,
  'os': {('David R.', 11): 1,
         ('David R.', 12): 1,
         ('David R.', 13): 1,
         ('David R.', 14): 1,
         ('David R.', 15): 1,
         ('Fritol', 8): 1,
         ('Fritol', 9): 1,
         ('Fritol', 11): 1,
         ('Fritol', 12): 1,
         ('Fritol', 15): 1,
         ('Jana', 9): 1,
         ('Jana', 11): 1,
         ('Jana', 12): 1,
         ('Jana', 13): 1,
         ('Jana', 14): 1,
         ('Jana', 15): 1,
         ('Josef', 8): 1,
         ('Josef', 9): 1,
         ('Josef', 11): 1,
         ('Josef', 12): 1,
         ('Josef', 13): 1,
         ('Josef', 14): 1,
         ('Josef', 15): 1,
         ('Martin Vitvar', 8): 1,
         ('Martin Vitvar', 9): 1,
         ('Martin Vitvar', 11): 1,
         ('Martin Vitvar', 12): 1,
         ('Martin Vitvar', 13): 1,
         ('Martin Vitvar', 14): 1,
         ('Martin Vitvar', 15): 1,
         ('Petr Hl', 8): 1,
         ('Petr Hl', 9): 1,
         ('Petr Hl', 12): 1,
         ('Petr Hl', 13): 1,
         ('Petr Hl', 14): 1,
         ('Petr Hl', 15): 1,
         ('Voldamic', 11): 1,
         ('Voldamic', 12): 1,
         ('Voldamic', 13): 1,
         ('Voldamic', 14): 1,
         ('Voldamic', 15): 1,
         ('budulinek', 8): 1,
         ('budulinek', 11): 1,
         ('budulinek', 12): 1,
         ('budulinek', 13): 1,
         ('budulinek', 14): 1,
         ('budulinek', 15): 1,
         ('cinklond', 8): 1,
         ('cinklond', 11): 1,
         ('cinklond', 12): 1,
         ('cinklond', 13): 1,
         ('cinklond', 14): 1,
         ('cinklond', 15): 1,
         ('delcamat', 11): 1,
         ('delcamat', 13): 1,
         ('delcamat', 14): 1,
         ('delcamat', 15): 1,
         ('dingo', 9): 1,
         ('dingo', 11): 1,
         ('dingo', 12): 1,
         ('dingo', 13): 1,
         ('dingo', 14): 1,
         ('dingo', 15): 1,
         ('dominik', 8): 1,
         ('dominik', 12): 1,
         ('dominik', 13): 1,
         ('dominik', 14): 1,
         ('dominik', 15): 1,
         ('eleanor', 11): 1,
         ('eleanor', 12): 1,
         ('eleanor', 13): 1,
         ('eleanor', 14): 1,
         ('eleanor', 15): 1,
         ('lukas', 11): 1,
         ('lukas', 12): 1,
         ('lukas', 13): 1,
         ('lukas', 14): 1,
         ('lukas', 15): 1,
         ('madrvojt', 8): 1,
         ('madrvojt', 11): 1,
         ('madrvojt', 12): 1,
         ('madrvojt', 13): 1,
         ('madrvojt', 14): 1,
         ('madrvojt', 15): 1},
  'path': 'parrots3.png'},
 {'asset_id': 11,
  'content_id': 1,
  'os': {('David R.', 12): 1,
         ('David R.', 13): 1,
         ('David R.', 14): 1,
         ('David R.', 15): 1,
         ('Fritol', 8): 1,
         ('Fritol', 14): 1,
         ('Fritol', 15): 1,
         ('Jana', 8): 1,
         ('Jana', 12): 1,
         ('Jana', 13): 1,
         ('Jana', 14): 1,
         ('Jana', 15): 1,
         ('Josef', 12): 1,
         ('Josef', 13): 1,
         ('Josef', 14): 1,
         ('Josef', 15): 1,
         ('Martin Vitvar', 12): 1,
         ('Martin Vitvar', 13): 1,
         ('Martin Vitvar', 14): 1,
         ('Martin Vitvar', 15): 1,
         ('Petr Hl', 8): 1,
         ('Petr Hl', 9): 1,
         ('Petr Hl', 10): 1,
         ('Petr Hl', 13): 1,
         ('Petr Hl', 14): 1,
         ('Petr Hl', 15): 1,
         ('Voldamic', 12): 1,
         ('Voldamic', 13): 1,
         ('Voldamic', 14): 1,
         ('Voldamic', 15): 1,
         ('budulinek', 13): 1,
         ('budulinek', 14): 1,
         ('budulinek', 15): 1,
         ('cinklond', 8): 1,
         ('cinklond', 12): 1,
         ('cinklond', 13): 1,
         ('cinklond', 14): 1,
         ('cinklond', 15): 1,
         ('delcamat', 8): 1,
         ('delcamat', 12): 1,
         ('delcamat', 13): 1,
         ('delcamat', 14): 1,
         ('dingo', 12): 1,
         ('dingo', 13): 1,
         ('dingo', 14): 1,
         ('dingo', 15): 1,
         ('dominik', 8): 1,
         ('dominik', 10): 1,
         ('dominik', 12): 1,
         ('dominik', 13): 1,
         ('dominik', 14): 1,
         ('dominik', 15): 1,
         ('eleanor', 13): 1,
         ('eleanor', 14): 1,
         ('eleanor', 15): 1,
         ('lukas', 12): 1,
         ('lukas', 13): 1,
         ('lukas', 14): 1,
         ('lukas', 15): 1,
         ('madrvojt', 13): 1,
         ('madrvojt', 14): 1,
         ('madrvojt', 15): 1},
  'path': 'parrots4.png'},
 {'asset_id': 12,
  'content_id': 1,
  'os': {('David R.', 8): 1,
         ('David R.', 13): 1,
         ('David R.', 14): 1,
         ('David R.', 15): 1,
         ('Fritol', 8): 1,
         ('Fritol', 11): 1,
         ('Fritol', 14): 1,
         ('Fritol', 15): 1,
         ('Jana', 13): 1,
         ('Jana', 14): 1,
         ('Jana', 15): 1,
         ('Josef', 13): 1,
         ('Josef', 14): 1,
         ('Josef', 15): 1,
         ('Martin Vitvar', 13): 1,
         ('Martin Vitvar', 14): 1,
         ('Martin Vitvar', 15): 1,
         ('Petr Hl', 11): 1,
         ('Petr Hl', 14): 1,
         ('Petr Hl', 15): 1,
         ('Voldamic', 13): 1,
         ('Voldamic', 15): 1,
         ('budulinek', 11): 1,
         ('budulinek', 14): 1,
         ('budulinek', 15): 1,
         ('cinklond', 8): 1,
         ('cinklond', 13): 1,
         ('cinklond', 14): 1,
         ('delcamat', 10): 1,
         ('delcamat', 13): 1,
         ('delcamat', 14): 1,
         ('delcamat', 15): 1,
         ('dingo', 13): 1,
         ('dingo', 14): 1,
         ('dingo', 15): 1,
         ('dominik', 13): 1,
         ('dominik', 14): 1,
         ('dominik', 15): 1,
         ('eleanor', 11): 1,
         ('eleanor', 14): 1,
         ('eleanor', 15): 1,
         ('lukas', 14): 1,
         ('lukas', 15): 1,
         ('madrvojt', 11): 1,
         ('madrvojt', 15): 1},
  'path': 'parrots5.png'},
 {'asset_id': 13,
  'content_id': 1,
  'os': {('David R.', 14): 1,
         ('David R.', 15): 1,
         ('Fritol', 8): 1,
         ('Fritol', 10): 1,
         ('Fritol', 11): 1,
         ('Fritol', 12): 1,
         ('Fritol', 14): 1,
         ('Jana', 14): 1,
         ('Jana', 15): 1,
         ('Josef', 14): 1,
         ('Josef', 15): 1,
         ('Martin Vitvar', 15): 1,
         ('Petr Hl', 8): 1,
         ('Petr Hl', 12): 1,
         ('Petr Hl', 14): 1,
         ('Petr Hl', 15): 1,
         ('Voldamic', 9): 1,
         ('Voldamic', 15): 1,
         ('budulinek', 12): 1,
         ('budulinek', 14): 1,
         ('cinklond', 15): 1,
         ('delcamat', 14): 1,
         ('delcamat', 15): 1,
         ('dingo', 15): 1,
         ('dominik', 14): 1,
         ('dominik', 15): 1,
         ('eleanor', 12): 1,
         ('eleanor', 14): 1,
         ('eleanor', 15): 1,
         ('lukas', 12): 1,
         ('lukas', 15): 1,
         ('madrvojt', 12): 1},
  'path': 'parrots6.png'},
 {'asset_id': 14,
  'content_id': 1,
  'os': {('David R.', 15): 1,
         ('Fritol', 10): 1,
         ('Jana', 15): 1,
         ('Josef', 15): 1,
         ('Martin Vitvar', 13): 1,
         ('Martin Vitvar', 15): 1,
         ('Petr Hl', 15): 1,
         ('Voldamic', 12): 1,
         ('Voldamic', 13): 1,
         ('Voldamic', 15): 1,
         ('budulinek', 15): 1,
         ('cinklond', 13): 1,
         ('cinklond', 15): 1,
         ('dingo', 13): 1,
         ('dingo', 15): 1,
         ('dominik', 15): 1,
         ('eleanor', 15): 1,
         ('lukas', 13): 1,
         ('lukas', 15): 1,
         ('madrvojt', 12): 1,
         ('madrvojt', 13): 1,
         ('madrvojt', 15): 1},
  'path': 'parrots7.png'},
 {'asset_id': 15,
  'content_id': 1,
  'os': {('Fritol', 9): 1,
         ('Fritol', 13): 1,
         ('Fritol', 14): 1,
         ('budulinek', 13): 1,
         ('cinklond', 12): 1,
         ('delcamat', 11): 1,
         ('delcamat', 14): 1,
         ('madrvojt', 13): 1},
  'path': 'parrots8.png'},
]

nest_site/nest/tests/resource/lukas_pc_dataset_tiny.py
Netflix/e2nest
dataset_name = 'VMAF_Plus_DB'
yuv_fmt = 'yuv420p'
width = 1920
height = 1080

# quality_width = 1280
# quality_height = 720
# resampling_type = 'lanczos'

# quality_width = 720
# quality_height = 480
# resampling_type = 'lanczos'

ref_score = 100.0

ref_videos = [{'content_id': 0,
  'content_name': 'BarbieLife_B',
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/ref_yuv/BarbieLife_B_ref.yuv'},
 {'content_id': 1,
  'content_name': 'BeautifulGirls_A',
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/ref_yuv/BeautifulGirls_A_ref.yuv'},
 {'content_id': 2,
  'content_name': 'Bloodline_A',
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/ref_yuv/Bloodline_A_ref.yuv'},
]

dis_videos = [{'asset_id': 1000,
  'content_id': 0,
  'os': {'andrewp': 89.0,
         'andyc': 80.0,
         'aschuler': 49.0,
         'asinghal': 68.0,
         'btorres': 100.0,
         'caleb': 88.0,
         'cwilms': 81.0,
         'czhao': 90.0,
         'ddenlinger': 79.0,
         'decsobiel@gmail.com': 96.0,
         'devenson@catflix.com': 96.0,
         'dteoh': 98.0,
         'hjagadeesan': 59.0,
         'jingxian@sranford.edu': 60.0,
         'kalvey': 94.0,
         'kchen@catflix.com': 96.0,
         'lfriedman@catflix.com': 84.0,
         'mangalap': 88.0,
         'mariakaz': 100.0,
         'mrobbe': 96.0,
         'msemeniakin': 48.0,
         'mufford': 64.0,
         'rbrodie': 100.0,
         'rgallardo': 92.0,
         'rly': 88.0,
         'rsharma': 92.0,
         'rwaters': 81.0,
         'sdearaujo': 91.0,
         'sdu@catflix.com': 96.0,
         'sfedorov@catflix.com': 95.0,
         'ssharma': 93.0,
         'svenkatrav@catflix.com': 58.0,
         'tgoodall@catflix.com': 90.0,
         'thuang': 67.0,
         'vyelevich': 96.0,
         'wwolcott': 94.0,
         'zchen': 88.0,
         'zli': 96.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/ref_yuv/BarbieLife_B_ref.yuv'},
 {'asset_id': 1001,
  'content_id': 1,
  'os': {'andrewp': 85.0,
         'andyc': 87.0,
         'aschuler': 77.0,
         'btorres': 41.0,
         'caleb': 93.0,
         'cwilms': 81.0,
         'czhao': 79.0,
         'ddenlinger': 74.0,
         'decsobiel@gmail.com': 56.0,
         'devenson@catflix.com': 78.0,
         'dteoh': 65.0,
         'hjagadeesan': 83.0,
         'jingxian@sranford.edu': 81.0,
         'kalvey': 100.0,
         'kchen@catflix.com': 73.0,
         'lfriedman@catflix.com': 84.0,
         'mangalap': 93.0,
         'mariakaz': 89.0,
         'mrobbe': 93.0,
         'msemeniakin': 91.0,
         'mufford': 64.0,
         'rbrodie': 93.0,
         'rgallardo': 85.0,
         'rly': 91.0,
         'rsharma': 63.0,
         'rwaters': 74.0,
         'sdearaujo': 86.0,
         'sdu@catflix.com': 74.0,
         'sfedorov@catflix.com': 85.0,
         'ssharma': 89.0,
         'svenkatrav@catflix.com': 51.0,
         'tgoodall@catflix.com': 100.0,
         'thuang': 76.0,
         'timothy.l.robinson@comcast.net': 76.0,
         'yjayaraman': 85.0,
         'zchen': 85.0,
         'zli': 66.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/ref_yuv/BeautifulGirls_A_ref.yuv'},
 {'asset_id': 1002,
  'content_id': 2,
  'os': {'andrewp': 98.0,
         'andyc': 82.0,
         'aprakash@catflix.com': 75.0,
         'aschuler': 82.0,
         'btorres': 95.0,
         'caleb': 97.0,
         'cwilms': 89.0,
         'czhao': 89.0,
         'ddenlinger': 68.0,
         'decsobiel@gmail.com': 92.0,
         'devenson@catflix.com': 61.0,
         'dteoh': 97.0,
         'hjagadeesan': 76.0,
         'jingxian@sranford.edu': 75.0,
         'kalvey': 99.0,
         'kchen@catflix.com': 91.0,
         'lfriedman@catflix.com': 70.0,
         'mangalap': 73.0,
         'mariakaz': 98.0,
         'mrobbe': 90.0,
         'msemeniakin': 96.0,
         'mufford': 86.0,
         'rbrodie': 98.0,
         'rgallardo': 91.0,
         'rly': 84.0,
         'rsharma': 91.0,
         'rwaters': 100.0,
         'sdearaujo': 93.0,
         'sdu@catflix.com': 90.0,
         'sfedorov@catflix.com': 98.0,
         'ssharma': 90.0,
         'thuang': 80.0,
         'timothy.l.robinson@comcast.net': 100.0,
         'vyelevich': 92.0,
         'wwolcott': 95.0,
         'yjayaraman': 81.0,
         'zchen': 89.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/ref_yuv/Bloodline_A_ref.yuv'},
 {'asset_id': 3,
  'content_id': 0,
  'os': {'andrewp': 80.0,
         'andyc': 65.0,
         'czhao': 92.0,
         'ddenlinger': 74.0,
         'devenson@catflix.com': 98.0,
         'dteoh': 80.0,
         'jingxian@sranford.edu': 53.0,
         'kalvey': 83.0,
         'lfriedman@catflix.com': 52.0,
         'mangalap': 36.0,
         'mrobbe': 95.0,
         'msemeniakin': 72.0,
         'rbrodie': 85.0,
         'rgallardo': 94.0,
         'rsharma': 91.0,
         'svenkatrav@catflix.com': 47.0,
         'wwolcott': 100.0,
         'zchen': 75.0,
         'zli': 89.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BarbieLife_B_fps23.97_1280x720_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 6,
  'content_id': 0,
  'os': {'aschuler': 53.0,
         'asinghal': 62.0,
         'btorres': 40.0,
         'caleb': 89.0,
         'cwilms': 82.0,
         'decsobiel@gmail.com': 97.0,
         'hjagadeesan': 78.0,
         'kchen@catflix.com': 98.0,
         'mariakaz': 86.0,
         'mufford': 64.0,
         'rly': 69.0,
         'rwaters': 79.0,
         'sdearaujo': 100.0,
         'sdu@catflix.com': 96.0,
         'sfedorov@catflix.com': 93.0,
         'ssharma': 83.0,
         'tgoodall@catflix.com': 100.0,
         'thuang': 52.0,
         'vyelevich': 89.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BarbieLife_B_fps23.97_1920x1080_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 11,
  'content_id': 0,
  'os': {'aschuler': 1.0,
         'caleb': 4.0,
         'ddenlinger': 22.0,
         'devenson@catflix.com': 11.0,
         'hjagadeesan': 10.0,
         'jingxian@sranford.edu': 1.0,
         'kalvey': 1.0,
         'kchen@catflix.com': 11.0,
         'lfriedman@catflix.com': 1.0,
         'mangalap': 17.0,
         'rbrodie': 1.0,
         'rly': 35.0,
         'rwaters': 1.0,
         'sdearaujo': 24.0,
         'sdu@catflix.com': 3.0,
         'sfedorov@catflix.com': 14.0,
         'svenkatrav@catflix.com': 9.0,
         'tgoodall@catflix.com': 1.0,
         'wwolcott': 32.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BarbieLife_B_fps23.97_320x240_x264_nflx3pass_28rf_bitrate10000.yuv'},
 {'asset_id': 13,
  'content_id': 0,
  'os': {'andrewp': 11.0,
         'andyc': 23.0,
         'asinghal': 20.0,
         'btorres': 3.0,
         'cwilms': 6.0,
         'czhao': 17.0,
         'decsobiel@gmail.com': 14.0,
         'dteoh': 4.0,
         'mariakaz': 15.0,
         'mrobbe': 1.0,
         'msemeniakin': 1.0,
         'mufford': 42.0,
         'rgallardo': 35.0,
         'rsharma': 49.0,
         'ssharma': 1.0,
         'thuang': 2.0,
         'vyelevich': 17.0,
         'zchen': 12.0,
         'zli': 12.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BarbieLife_B_fps23.97_384x288_x264_nflx3pass_28rf_bitrate10000.yuv'},
 {'asset_id': 14,
  'content_id': 0,
  'os': {'aschuler': 18.0,
         'caleb': 79.0,
         'cwilms': 35.0,
         'ddenlinger': 55.0,
         'decsobiel@gmail.com': 74.0,
         'hjagadeesan': 58.0,
         'jingxian@sranford.edu': 33.0,
         'kalvey': 74.0,
         'kchen@catflix.com': 41.0,
         'mangalap': 32.0,
         'mariakaz': 81.0,
         'rgallardo': 71.0,
         'rly': 75.0,
         'rwaters': 29.0,
         'sdearaujo': 65.0,
         'sfedorov@catflix.com': 74.0,
         'ssharma': 24.0,
         'svenkatrav@catflix.com': 52.0,
         'tgoodall@catflix.com': 75.0,
         'vyelevich': 38.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BarbieLife_B_fps23.97_512x384_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 15,
  'content_id': 0,
  'os': {'asinghal': 58.0,
         'btorres': 16.0,
         'caleb': 44.0,
         'cwilms': 43.0,
         'dteoh': 38.0,
         'jingxian@sranford.edu': 42.0,
         'lfriedman@catflix.com': 61.0,
         'mariakaz': 49.0,
         'mrobbe': 39.0,
         'msemeniakin': 62.0,
         'mufford': 43.0,
         'rbrodie': 2.0,
         'rgallardo': 63.0,
         'rly': 65.0,
         'rwaters': 25.0,
         'sdearaujo': 53.0,
         'svenkatrav@catflix.com': 47.0,
         'vyelevich': 41.0,
         'zchen': 41.0,
         'zli': 52.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BarbieLife_B_fps23.97_512x384_x264_nflx3pass_25rf_bitrate10000.yuv'},
 {'asset_id': 16,
  'content_id': 0,
  'os': {'andrewp': 22.0,
         'andyc': 32.0,
         'aschuler': 5.0,
         'btorres': 27.0,
         'czhao': 34.0,
         'devenson@catflix.com': 31.0,
         'hjagadeesan': 29.0,
         'kalvey': 34.0,
         'kchen@catflix.com': 31.0,
         'lfriedman@catflix.com': 1.0,
         'mangalap': 57.0,
         'rbrodie': 15.0,
         'rsharma': 43.0,
         'sdu@catflix.com': 35.0,
         'sfedorov@catflix.com': 24.0,
         'svenkatrav@catflix.com': 22.0,
         'thuang': 3.0,
         'wwolcott': 19.0,
         'zli': 43.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BarbieLife_B_fps23.97_512x384_x264_nflx3pass_28rf_bitrate10000.yuv'},
 {'asset_id': 17,
  'content_id': 0,
  'os': {'andrewp': 65.0,
         'andyc': 65.0,
         'aschuler': 33.0,
         'asinghal': 75.0,
         'caleb': 88.0,
         'ddenlinger': 57.0,
         'decsobiel@gmail.com': 64.0,
         'devenson@catflix.com': 85.0,
         'hjagadeesan': 61.0,
         'kchen@catflix.com': 87.0,
         'mrobbe': 50.0,
         'mufford': 51.0,
         'rbrodie': 78.0,
         'rwaters': 78.0,
         'sdu@catflix.com': 62.0,
         'thuang': 13.0,
         'vyelevich': 86.0,
         'zli': 79.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BarbieLife_B_fps23.97_720x480_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 18,
  'content_id': 0,
  'os': {'andrewp': 58.0,
         'asinghal': 59.0,
         'btorres': 40.0,
         'czhao': 63.0,
         'decsobiel@gmail.com': 64.0,
         'devenson@catflix.com': 70.0,
         'dteoh': 30.0,
         'mangalap': 27.0,
         'msemeniakin': 64.0,
         'rgallardo': 64.0,
         'rly': 83.0,
         'rsharma': 71.0,
         'sfedorov@catflix.com': 79.0,
         'ssharma': 16.0,
         'tgoodall@catflix.com': 81.0,
         'thuang': 9.0,
         'wwolcott': 42.0,
         'zchen': 53.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BarbieLife_B_fps23.97_720x480_x264_nflx3pass_25rf_bitrate10000.yuv'},
 {'asset_id': 19,
  'content_id': 0,
  'os': {'andyc': 53.0,
         'cwilms': 26.0,
         'czhao': 62.0,
         'ddenlinger': 63.0,
         'dteoh': 29.0,
         'jingxian@sranford.edu': 33.0,
         'kalvey': 37.0,
         'lfriedman@catflix.com': 31.0,
         'mariakaz': 39.0,
         'mrobbe': 19.0,
         'msemeniakin': 49.0,
         'mufford': 48.0,
         'rsharma': 62.0,
         'sdearaujo': 83.0,
         'sdu@catflix.com': 17.0,
         'ssharma': 22.0,
         'tgoodall@catflix.com': 61.0,
         'wwolcott': 75.0,
         'zchen': 48.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BarbieLife_B_fps23.97_720x480_x264_nflx3pass_28rf_bitrate10000.yuv'},
 {'asset_id': 20,
  'content_id': 1,
  'os': {'andrewp': 83.0,
         'andyc': 82.0,
         'czhao': 71.0,
         'ddenlinger': 77.0,
         'devenson@catflix.com': 74.0,
         'dteoh': 76.0,
         'jingxian@sranford.edu': 55.0,
         'kalvey': 100.0,
         'lfriedman@catflix.com': 89.0,
         'mangalap': 62.0,
         'mrobbe': 90.0,
         'msemeniakin': 56.0,
         'rbrodie': 95.0,
         'rgallardo': 94.0,
         'rsharma': 84.0,
         'svenkatrav@catflix.com': 64.0,
         'yjayaraman': 93.0,
         'zchen': 83.0,
         'zli': 50.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BeautifulGirls_A_fps23.97_1280x720_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 23,
  'content_id': 1,
  'os': {'aschuler': 72.0,
         'btorres': 84.0,
         'caleb': 97.0,
         'cwilms': 83.0,
         'decsobiel@gmail.com': 80.0,
         'hjagadeesan': 81.0,
         'kchen@catflix.com': 100.0,
         'mariakaz': 100.0,
         'mufford': 56.0,
         'rly': 79.0,
         'rwaters': 43.0,
         'sdearaujo': 88.0,
         'sdu@catflix.com': 92.0,
         'sfedorov@catflix.com': 97.0,
         'ssharma': 83.0,
         'tgoodall@catflix.com': 80.0,
         'thuang': 77.0,
         'timothy.l.robinson@comcast.net': 64.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BeautifulGirls_A_fps23.97_1920x1080_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 27,
  'content_id': 1,
  'os': {'aschuler': 6.0,
         'caleb': 4.0,
         'ddenlinger': 21.0,
         'devenson@catflix.com': 35.0,
         'hjagadeesan': 10.0,
         'jingxian@sranford.edu': 11.0,
         'kalvey': 1.0,
         'kchen@catflix.com': 11.0,
         'lfriedman@catflix.com': 16.0,
         'mangalap': 1.0,
         'rbrodie': 14.0,
         'rly': 32.0,
         'rwaters': 1.0,
         'sdearaujo': 17.0,
         'sdu@catflix.com': 3.0,
         'sfedorov@catflix.com': 8.0,
         'svenkatrav@catflix.com': 5.0,
         'tgoodall@catflix.com': 1.0,
         'timothy.l.robinson@comcast.net': 19.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BeautifulGirls_A_fps23.97_320x240_x264_nflx3pass_28rf_bitrate10000.yuv'},
 {'asset_id': 30,
  'content_id': 1,
  'os': {'andrewp': 8.0,
         'andyc': 28.0,
         'btorres': 1.0,
         'cwilms': 6.0,
         'czhao': 18.0,
         'decsobiel@gmail.com': 11.0,
         'dteoh': 1.0,
         'mariakaz': 9.0,
         'mrobbe': 13.0,
         'msemeniakin': 15.0,
         'mufford': 20.0,
         'rgallardo': 28.0,
         'rsharma': 9.0,
         'ssharma': 21.0,
         'thuang': 24.0,
         'yjayaraman': 26.0,
         'zchen': 14.0,
         'zli': 11.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BeautifulGirls_A_fps23.97_384x288_x264_nflx3pass_28rf_bitrate10000.yuv'},
 {'asset_id': 31,
  'content_id': 1,
  'os': {'aschuler': 25.0,
         'caleb': 68.0,
         'cwilms': 43.0,
         'ddenlinger': 51.0,
         'decsobiel@gmail.com': 37.0,
         'hjagadeesan': 67.0,
         'jingxian@sranford.edu': 32.0,
         'kalvey': 64.0,
         'kchen@catflix.com': 68.0,
         'mangalap': 46.0,
         'mariakaz': 76.0,
         'rgallardo': 72.0,
         'rly': 70.0,
         'rwaters': 37.0,
         'sdearaujo': 48.0,
         'sfedorov@catflix.com': 41.0,
         'ssharma': 54.0,
         'svenkatrav@catflix.com': 38.0,
         'tgoodall@catflix.com': 61.0,
         'yjayaraman': 74.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BeautifulGirls_A_fps23.97_512x384_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 32,
  'content_id': 1,
  'os': {'btorres': 18.0,
         'caleb': 68.0,
         'cwilms': 19.0,
         'dteoh': 33.0,
         'jingxian@sranford.edu': 22.0,
         'lfriedman@catflix.com': 15.0,
         'mariakaz': 46.0,
         'mrobbe': 26.0,
         'msemeniakin': 26.0,
         'mufford': 42.0,
         'rbrodie': 34.0,
         'rgallardo': 71.0,
         'rly': 61.0,
         'rwaters': 14.0,
         'sdearaujo': 51.0,
         'svenkatrav@catflix.com': 20.0,
         'timothy.l.robinson@comcast.net': 30.0,
         'zchen': 33.0,
         'zli': 41.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BeautifulGirls_A_fps23.97_512x384_x264_nflx3pass_25rf_bitrate10000.yuv'},
 {'asset_id': 33,
  'content_id': 1,
  'os': {'andrewp': 32.0,
         'andyc': 34.0,
         'aschuler': 23.0,
         'btorres': 27.0,
         'czhao': 29.0,
         'devenson@catflix.com': 39.0,
         'hjagadeesan': 28.0,
         'kalvey': 26.0,
         'kchen@catflix.com': 64.0,
         'lfriedman@catflix.com': 24.0,
         'mangalap': 48.0,
         'rbrodie': 14.0,
         'rsharma': 36.0,
         'sdu@catflix.com': 6.0,
         'sfedorov@catflix.com': 40.0,
         'svenkatrav@catflix.com': 14.0,
         'thuang': 11.0,
         'timothy.l.robinson@comcast.net': 60.0,
         'zli': 28.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BeautifulGirls_A_fps23.97_512x384_x264_nflx3pass_28rf_bitrate10000.yuv'},
 {'asset_id': 34,
  'content_id': 1,
  'os': {'andrewp': 49.0,
         'andyc': 50.0,
         'aschuler': 39.0,
         'caleb': 98.0,
         'ddenlinger': 49.0,
         'decsobiel@gmail.com': 66.0,
         'devenson@catflix.com': 71.0,
         'hjagadeesan': 68.0,
         'kchen@catflix.com': 57.0,
         'mrobbe': 59.0,
         'mufford': 55.0,
         'rbrodie': 98.0,
         'rwaters': 51.0,
         'sdu@catflix.com': 96.0,
         'thuang': 48.0,
         'timothy.l.robinson@comcast.net': 48.0,
         'zli': 63.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BeautifulGirls_A_fps23.97_720x480_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 35,
  'content_id': 1,
  'os': {'andrewp': 57.0,
         'btorres': 25.0,
         'czhao': 48.0,
         'decsobiel@gmail.com': 49.0,
         'devenson@catflix.com': 80.0,
         'dteoh': 67.0,
         'mangalap': 71.0,
         'msemeniakin': 81.0,
         'rgallardo': 83.0,
         'rly': 55.0,
         'rsharma': 72.0,
         'sfedorov@catflix.com': 77.0,
         'ssharma': 50.0,
         'tgoodall@catflix.com': 83.0,
         'thuang': 38.0,
         'yjayaraman': 55.0,
         'zchen': 49.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/BeautifulGirls_A_fps23.97_720x480_x264_nflx3pass_25rf_bitrate10000.yuv'},
 {'asset_id': 36,
  'content_id': 2,
  'os': {'andrewp': 83.0,
         'andyc': 94.0,
         'czhao': 91.0,
         'ddenlinger': 76.0,
         'devenson@catflix.com': 96.0,
         'dteoh': 85.0,
         'jingxian@sranford.edu': 63.0,
         'kalvey': 100.0,
         'lfriedman@catflix.com': 81.0,
         'mangalap': 49.0,
         'mrobbe': 93.0,
         'msemeniakin': 89.0,
         'rbrodie': 99.0,
         'rgallardo': 93.0,
         'rsharma': 92.0,
         'wwolcott': 73.0,
         'yjayaraman': 98.0,
         'zchen': 88.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/Bloodline_A_fps23.97_1280x720_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 39,
  'content_id': 2,
  'os': {'aprakash@catflix.com': 85.0,
         'aschuler': 72.0,
         'btorres': 91.0,
         'caleb': 52.0,
         'cwilms': 92.0,
         'decsobiel@gmail.com': 86.0,
         'hjagadeesan': 72.0,
         'kchen@catflix.com': 100.0,
         'mariakaz': 100.0,
         'mufford': 70.0,
         'rly': 90.0,
         'rwaters': 100.0,
         'sdearaujo': 85.0,
         'sdu@catflix.com': 98.0,
         'sfedorov@catflix.com': 98.0,
         'ssharma': 91.0,
         'thuang': 49.0,
         'timothy.l.robinson@comcast.net': 90.0,
         'vyelevich': 89.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/Bloodline_A_fps23.97_1920x1080_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 44,
  'content_id': 2,
  'os': {'aprakash@catflix.com': 41.0,
         'aschuler': 8.0,
         'caleb': 7.0,
         'ddenlinger': 21.0,
         'devenson@catflix.com': 18.0,
         'hjagadeesan': 5.0,
         'jingxian@sranford.edu': 1.0,
         'kalvey': 1.0,
         'kchen@catflix.com': 14.0,
         'lfriedman@catflix.com': 10.0,
         'mangalap': 1.0,
         'rbrodie': 2.0,
         'rly': 28.0,
         'rwaters': 1.0,
         'sdearaujo': 20.0,
         'sdu@catflix.com': 1.0,
         'sfedorov@catflix.com': 7.0,
         'timothy.l.robinson@comcast.net': 10.0,
         'wwolcott': 1.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/Bloodline_A_fps23.97_320x240_x264_nflx3pass_28rf_bitrate10000.yuv'},
 {'asset_id': 46,
  'content_id': 2,
  'os': {'andrewp': 19.0,
         'andyc': 24.0,
         'btorres': 13.0,
         'cwilms': 5.0,
         'czhao': 17.0,
         'decsobiel@gmail.com': 13.0,
         'dteoh': 1.0,
         'mariakaz': 39.0,
         'mrobbe': 3.0,
         'msemeniakin': 34.0,
         'mufford': 24.0,
         'rgallardo': 24.0,
         'rsharma': 38.0,
         'ssharma': 1.0,
         'thuang': 7.0,
         'vyelevich': 16.0,
         'yjayaraman': 15.0,
         'zchen': 12.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/Bloodline_A_fps23.97_384x288_x264_nflx3pass_28rf_bitrate10000.yuv'},
 {'asset_id': 47,
  'content_id': 2,
  'os': {'aprakash@catflix.com': 63.0,
         'aschuler': 21.0,
         'caleb': 68.0,
         'cwilms': 52.0,
         'ddenlinger': 47.0,
         'decsobiel@gmail.com': 41.0,
         'hjagadeesan': 63.0,
         'jingxian@sranford.edu': 28.0,
         'kalvey': 100.0,
         'kchen@catflix.com': 86.0,
         'mangalap': 55.0,
         'mariakaz': 73.0,
         'rgallardo': 82.0,
         'rly': 67.0,
         'rwaters': 40.0,
         'sdearaujo': 49.0,
         'sfedorov@catflix.com': 45.0,
         'ssharma': 47.0,
         'vyelevich': 42.0,
         'yjayaraman': 74.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/Bloodline_A_fps23.97_512x384_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 48,
  'content_id': 2,
  'os': {'btorres': 46.0,
         'caleb': 54.0,
         'cwilms': 35.0,
         'dteoh': 11.0,
         'jingxian@sranford.edu': 39.0,
         'lfriedman@catflix.com': 31.0,
         'mariakaz': 52.0,
         'mrobbe': 19.0,
         'msemeniakin': 54.0,
         'mufford': 64.0,
         'rbrodie': 14.0,
         'rgallardo': 74.0,
         'rly': 63.0,
         'rwaters': 43.0,
         'sdearaujo': 51.0,
         'timothy.l.robinson@comcast.net': 39.0,
         'vyelevich': 39.0,
         'zchen': 22.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/Bloodline_A_fps23.97_512x384_x264_nflx3pass_25rf_bitrate10000.yuv'},
 {'asset_id': 49,
  'content_id': 2,
  'os': {'andrewp': 22.0,
         'andyc': 27.0,
         'aprakash@catflix.com': 47.0,
         'aschuler': 3.0,
         'btorres': 20.0,
         'czhao': 52.0,
         'devenson@catflix.com': 33.0,
         'hjagadeesan': 40.0,
         'kalvey': 62.0,
         'kchen@catflix.com': 22.0,
         'lfriedman@catflix.com': 44.0,
         'mangalap': 19.0,
         'rbrodie': 15.0,
         'rsharma': 37.0,
         'sdu@catflix.com': 7.0,
         'sfedorov@catflix.com': 52.0,
         'thuang': 12.0,
         'timothy.l.robinson@comcast.net': 21.0,
         'wwolcott': 27.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/Bloodline_A_fps23.97_512x384_x264_nflx3pass_28rf_bitrate10000.yuv'},
 {'asset_id': 50,
  'content_id': 2,
  'os': {'andrewp': 62.0,
         'andyc': 79.0,
         'aschuler': 30.0,
         'caleb': 97.0,
         'ddenlinger': 49.0,
         'decsobiel@gmail.com': 43.0,
         'devenson@catflix.com': 69.0,
         'hjagadeesan': 58.0,
         'kchen@catflix.com': 69.0,
         'mrobbe': 58.0,
         'mufford': 85.0,
         'rbrodie': 74.0,
         'rwaters': 75.0,
         'sdu@catflix.com': 44.0,
         'thuang': 49.0,
         'timothy.l.robinson@comcast.net': 58.0,
         'vyelevich': 75.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/Bloodline_A_fps23.97_720x480_x264_nflx3pass_22rf_bitrate10000.yuv'},
 {'asset_id': 51,
  'content_id': 2,
  'os': {'andrewp': 47.0,
         'aprakash@catflix.com': 76.0,
         'btorres': 27.0,
         'czhao': 68.0,
         'decsobiel@gmail.com': 64.0,
         'devenson@catflix.com': 88.0,
         'dteoh': 48.0,
         'mangalap': 34.0,
         'msemeniakin': 83.0,
         'rgallardo': 90.0,
         'rly': 87.0,
         'rsharma': 69.0,
         'sfedorov@catflix.com': 66.0,
         'ssharma': 67.0,
         'thuang': 42.0,
         'wwolcott': 78.0,
         'yjayaraman': 71.0,
         'zchen': 47.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/Bloodline_A_fps23.97_720x480_x264_nflx3pass_25rf_bitrate10000.yuv'},
 {'asset_id': 52,
  'content_id': 2,
  'os': {'andyc': 46.0,
         'cwilms': 21.0,
         'czhao': 61.0,
         'ddenlinger': 36.0,
         'dteoh': 20.0,
         'jingxian@sranford.edu': 47.0,
         'kalvey': 80.0,
         'lfriedman@catflix.com': 55.0,
         'mariakaz': 48.0,
         'mrobbe': 29.0,
         'msemeniakin': 47.0,
         'mufford': 59.0,
         'rsharma': 77.0,
         'sdearaujo': 52.0,
         'sdu@catflix.com': 22.0,
         'ssharma': 40.0,
         'wwolcott': 68.0,
         'yjayaraman': 98.0,
         'zchen': 37.0},
  'path': '/mnt/hgfs/zli-nflx3/VMAF_Plus_study/dis_yuv/Bloodline_A_fps23.97_720x480_x264_nflx3pass_28rf_bitrate10000.yuv'},
]

nest_site/nest/tests/resource/vmafplusstudy_laptop_raw_tiny.py
Netflix/e2nest
from django.contrib.auth.views import LogoutView, PasswordChangeDoneView, \
    PasswordChangeView


class NestLogoutView(LogoutView):
    template_name = 'nest/logged_out.html'


class NestPasswordChangeView(PasswordChangeView):
    template_name = 'nest/password_change_form.html'


class NestPasswordChangeDoneView(PasswordChangeDoneView):
    template_name = 'nest/password_change_done.html'

nest_site/nest/views.py
Netflix/e2nest
"""
ASGI config for nest_site project.

It exposes the ASGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/
"""

import os

from django.core.asgi import get_asgi_application

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nest_site.settings')

application = get_asgi_application()

nest_site/nest_site/asgi.py
Netflix/e2nest
"""
Django settings for nest_site project.

Generated by 'django-admin startproject' using Django 3.0.8.

For more information on this file, see
https://docs.djangoproject.com/en/3.0/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.0/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "PUT_YOUR_SECRET_KEY_HERE"

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = [
    "localhost",
    "127.0.0.1",  # localhost
    "PUT_YOUR_SERVER_IP_HERE"
]


# Application definition

INSTALLED_APPS = [
    'polls.apps.PollsConfig',
    'nest.apps.NestConfig',
    'nest.apps.NestAdminConfig',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'mathfilters',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'nest_site.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(BASE_DIR, 'templates')],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'nest_site.wsgi.application'


# Database
# https://docs.djangoproject.com/en/3.0/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    # {
    #     'NAME': 'django.contrib.auth.password_validation'
    #             '.UserAttributeSimilarityValidator',
    # },
    # {
    #     'NAME': 'django.contrib.auth.password_validation'
    #             '.MinimumLengthValidator',
    # },
    # {
    #     'NAME': 'django.contrib.auth.password_validation'
    #             '.CommonPasswordValidator',
    # },
    # {
    #     'NAME': 'django.contrib.auth.password_validation'
    #             '.NumericPasswordValidator',
    # },
]


# Internationalization
# https://docs.djangoproject.com/en/3.0/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True


# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.0/howto/static-files/

STATIC_URL = '/static/'
# STATIC_ROOT = 'static/'
# STATICFILES_DIRS = [
#     os.path.join(BASE_DIR, 'nest', 'static')
# ]

MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')


def map_media_url_to_local(url):
    rel_path = os.path.relpath(url, MEDIA_URL)
    return os.path.join(MEDIA_ROOT, rel_path)


def map_media_local_to_url(local):
    rel_path = os.path.relpath(local, MEDIA_ROOT)
    return os.path.join(MEDIA_URL, rel_path)


# fixing warning of 'django.db.models.BigAutoField':
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'


CHROMEDRIVER_PATH = '/var/chromedriver'

nest_site/nest_site/settings.py
Netflix/e2nest
"""nest_site URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
import os

import nest
from django.contrib import admin
from django.contrib.staticfiles.storage import staticfiles_storage
from django.urls import include, re_path
from django.urls import path
from django.views.generic import RedirectView
from nest_site import settings
from third_party.ranged_response import RangedFileResponse

favicon_view = RedirectView.as_view(
    url=staticfiles_storage.url('nest/images/favicon.ico'), permanent=False)


def mp4_byterange_view(request, path):
    full_path = os.path.join(settings.MEDIA_ROOT, 'mp4', path)
    response = RangedFileResponse(request, open(full_path, 'rb'), content_type='video/mp4')
    response['Content-Disposition'] = f'attachment; filename="{full_path}"'
    return response


urlpatterns = [
    path('admin/', admin.site.urls, name='admin'),
    path('polls/', include('polls.urls'), name='polls'),
    path('', nest.site.urls, name='nest'),
    re_path(r'^favicon\.ico$', favicon_view, name='favicon'),
    re_path(r'^media/mp4/(?P<path>.*)$', mp4_byterange_view, name='mp4'),
]

nest_site/nest_site/urls.py
Netflix/e2nest
"""
WSGI config for nest_site project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/
"""

import os

from django.core.wsgi import get_wsgi_application

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nest_site.settings')

application = get_wsgi_application()

nest_site/nest_site/wsgi.py
Netflix/e2nest
from django.contrib import admin

from .models import Choice, Question


# class ChoiceInline(admin.StackedInline):
class ChoiceInline(admin.TabularInline):
    model = Choice
    extra = 0


class QuestionAdmin(admin.ModelAdmin):
    # fields = ['pub_date', 'question_text']
    fieldsets = [
        (None,               {'fields': ['question_text']}),
        ('Date information', {'fields': ['pub_date']}),
    ]
    inlines = [ChoiceInline]
    list_display = ('question_text', 'pub_date', 'was_published_recently')
    list_filter = ['pub_date']
    search_fields = ['question_text']


admin.site.register(Question, QuestionAdmin)


# admin.site.register(Question)
# admin.site.register(Choice)

nest_site/polls/admin.py
Netflix/e2nest
import datetime

from django.db import models
from django.utils import timezone
from polymorphic.models import PolymorphicModel


class Question(PolymorphicModel):
    question_text = models.CharField(max_length=200)
    pub_date = models.DateTimeField('date published')

    def __str__(self):
        return self.question_text

    def was_published_recently(self):
        now = timezone.now()
        return now >= self.pub_date >= now - datetime.timedelta(days=1)
    was_published_recently.admin_order_field = 'pub_date'
    was_published_recently.boolean = True
    was_published_recently.short_description = 'Published recently?'


class Choice(PolymorphicModel):

    # By "ForeignKey", Question now has a method called "choice_set"
    # and Choice now has a method called "question"
    question = models.ForeignKey(Question, on_delete=models.CASCADE)

    choice_text = models.CharField(max_length=200)
    votes = models.IntegerField(default=0)

    def __str__(self):
        return self.choice_text

nest_site/polls/models.py
Netflix/e2nest
import datetime

from django.test import TestCase
from django.urls import reverse
from django.utils import timezone

from .models import Choice, Question


class TestModels(TestCase):

    def test_question_and_choice(self):
        self.assertEqual(len(Question.objects.all()), 0)
        q = Question(question_text="What's new?", pub_date=timezone.now())
        q.save()
        self.assertEqual(len(Question.objects.all()), 1)
        self.assertEqual(q.question_text, "What's new?")
        self.assertTrue(Question.objects.get(pub_date__year=timezone.now().
                                             year).was_published_recently())
        q.choice_set.create(choice_text='Not much', votes=0)
        q.choice_set.create(choice_text='The sky', votes=0)
        self.assertEqual(len(q.choice_set.all()), 2)
        self.assertEqual(q.choice_set.count(), 2)
        c = q.choice_set.create(choice_text='Just hacking again', votes=0)
        self.assertEqual(c.question, q)
        c = q.choice_set.filter(choice_text__startswith='Just hacking')
        self.assertEqual(len(c), 1)
        self.assertEqual(len(Choice.objects.all()), 3)
        c.delete()
        self.assertEqual(len(c), 0)
        self.assertEqual(len(Choice.objects.all()), 2)

    def test_was_published_recently_with_future_question(self):
        """
        was_published_recently() returns False for questions whose pub_date
        is in the future.
        """
        time = timezone.now() + datetime.timedelta(days=30)
        future_question = Question(pub_date=time)
        self.assertIs(future_question.was_published_recently(), False)

    def test_was_published_recently_with_old_question(self):
        """
        was_published_recently() returns False for questions whose pub_date
        is older than 1 day.
        """
        time = timezone.now() - datetime.timedelta(days=1, seconds=1)
        old_question = Question(pub_date=time)
        self.assertIs(old_question.was_published_recently(), False)

    def test_was_published_recently_with_recent_question(self):
        """
        was_published_recently() returns True for questions whose pub_date
        is within the last day.
        """
        time = timezone.now() - datetime.timedelta(hours=23, minutes=59,
                                                   seconds=59)
        recent_question = Question(pub_date=time)
        self.assertIs(recent_question.was_published_recently(), True)


def create_question(question_text, days):
    """
    Create a question with the given `question_text` and published the
    given number of `days` offset to now (negative for questions published
    in the past, positive for questions that have yet to be published).
    """
    time = timezone.now() + datetime.timedelta(days=days)
    return Question.objects.create(question_text=question_text, pub_date=time)


class TestViews(TestCase):

    def test_404(self):
        response = self.client.get('/nonexist')
        self.assertEqual(response.status_code, 404)

    def test_main(self):
        response = self.client.get('/')
        self.assertEqual(response.status_code, 302)

    def test_polls(self):
        response = self.client.get(reverse('polls:index'))
        self.assertEqual(response.status_code, 200)

    def test_polls0(self):
        response = self.client.get(reverse('polls:detail', kwargs={'pk': 1}))
        self.assertEqual(response.status_code, 404)
        create_question(question_text="Past question.", days=-30)
        response = self.client.get(reverse('polls:detail', kwargs={'pk': 1}))
        self.assertEqual(response.status_code, 200)


class QuestionIndexViewTests(TestCase):
    def test_no_questions(self):
        """
        If no questions exist, an appropriate message is displayed.
        """
        response = self.client.get(reverse('polls:index'))
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, "No polls are available.")
        self.assertQuerysetEqual(response.context['latest_question_list'], [])

    def test_past_question(self):
        """
        Questions with a pub_date in the past are displayed on the
        index page.
        """
        create_question(question_text="Past question.", days=-30)
        response = self.client.get(reverse('polls:index'))
        self.assertQuerysetEqual(
            response.context['latest_question_list'],
            ['<Question: Past question.>']
        )

    def test_future_question(self):
        """
        Questions with a pub_date in the future aren't displayed on
        the index page.
        """
        create_question(question_text="Future question.", days=30)
        response = self.client.get(reverse('polls:index'))
        self.assertContains(response, "No polls are available.")
        self.assertQuerysetEqual(response.context['latest_question_list'], [])

    def test_future_question_and_past_question(self):
        """
        Even if both past and future questions exist, only past questions
        are displayed.
        """
        create_question(question_text="Past question.", days=-30)
        create_question(question_text="Future question.", days=30)
        response = self.client.get(reverse('polls:index'))
        self.assertQuerysetEqual(
            response.context['latest_question_list'],
            ['<Question: Past question.>']
        )

    def test_two_past_questions(self):
        """
        The questions index page may display multiple questions.
        """
        create_question(question_text="Past question 1.", days=-30)
        create_question(question_text="Past question 2.", days=-5)
        response = self.client.get(reverse('polls:index'))
        self.assertQuerysetEqual(
            response.context['latest_question_list'],
            ['<Question: Past question 2.>', '<Question: Past question 1.>']
        )

nest_site/polls/tests.py
Netflix/e2nest
from setuptools import setup


setup(
    name="e2nest",
    setup_requires="setupmeta",
    versioning="distance",
    author="Zhi Li (zli@netflix.com), Lukas Krasula (lkrasula@netflix.com)",
    url="https://github.com/Netflix/e2nest",
    py_modules=['nest_site'],
)

setup.py
Netflix/e2nest